{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BipedalWalker-v2 with PPO, Vectorized Environment\n",
    "In this notebook, you will implement a PPO agent with OpenAI Gym's BipedalWalker-v2 environment.\n",
    "\n",
    "### 1. Create Vectorized Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gym version:  0.13.1\n",
      "torch version:  1.0.1\n",
      "device:  cuda:0\n",
      "max_steps:  1600\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from  collections  import deque\n",
    "import time\n",
    "from model import Policy\n",
    "from ppo import ppo_agent\n",
    "from storage import RolloutStorage\n",
    "from utils import get_render_func, get_vec_normalize\n",
    "from envs import make_vec_envs\n",
    "from parallelEnv import parallelEnv\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('gym version: ', gym.__version__)\n",
    "print('torch version: ', torch.__version__)\n",
    "\n",
    "seed = 0 \n",
    "gamma=0.99\n",
    "num_processes=16 \n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print('device: ', device)\n",
    "\n",
    "envs = parallelEnv('BipedalWalker-v2', n=num_processes, seed=seed)\n",
    "\n",
    "## make_vec_envs -cannot find context for 'forkserver'\n",
    "## forkserver is only available in Python 3.4+ and only on some Unix platforms (not on Windows).\n",
    "## envs = make_vec_envs('BipedalWalker-v2', \\\n",
    "##                    seed + 1000, num_processes,\n",
    "##                    None, None, False, device='cpu', allow_early_resets=False)\n",
    "\n",
    "max_steps = envs.max_steps\n",
    "print('max_steps: ', max_steps)\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "np.random.seed(seed)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Instantiate Model, Agent and Storage\n",
    "\n",
    "Initialize the Policy (model MLPBase), PPO Agent and Rollout Storage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type obs:  <class 'numpy.ndarray'> , shape obs:  (16, 24)\n",
      "type obs_t:  <class 'torch.Tensor'> , shape obs_t:  torch.Size([16, 24])\n"
     ]
    }
   ],
   "source": [
    "## model Policy uses MLPBase\n",
    "policy = Policy(envs.observation_space.shape, envs.action_space,\\\n",
    "        base_kwargs={'recurrent': False})\n",
    "\n",
    "policy.to(device)\n",
    "\n",
    "agent = ppo_agent(actor_critic=policy, ppo_epoch=16, num_mini_batch=16,\\\n",
    "                lr=0.001, eps=1e-5, max_grad_norm=0.5)\n",
    "\n",
    "rollouts = RolloutStorage(num_steps=max_steps, num_processes=num_processes, \\\n",
    "                        obs_shape=envs.observation_space.shape, action_space=envs.action_space, \\\n",
    "                        recurrent_hidden_state_size=policy.recurrent_hidden_state_size)\n",
    "\n",
    "obs = envs.reset()\n",
    "print('type obs: ', type(obs), ', shape obs: ', obs.shape)\n",
    "obs_t = torch.tensor(obs)\n",
    "print('type obs_t: ', type(obs_t), ', shape obs_t: ', obs_t.shape)\n",
    "\n",
    "rollouts.obs[0].copy_(obs_t)\n",
    "rollouts.to(device)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.Save model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(model, directory, filename, suffix):\n",
    "    torch.save(model.base.actor.state_dict(), '%s/%s_actor_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(model.base.critic.state_dict(), '%s/%s_critic_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(model.base.critic_linear.state_dict(), '%s/%s_critic_linear_%s.pth' % (directory, filename, suffix))\n",
    "    \n",
    "limits = [-300, -160, -100, -70, -50, 0, 20, 30, 40, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330]\n",
    "\n",
    "def return_suffix(j):\n",
    "    suf = '0'\n",
    "    for i in range(len(limits)-1):\n",
    "        if j > limits[i] and j < limits[i+1]:\n",
    "            suf = str(limits[i+1])\n",
    "            break\n",
    "        \n",
    "        i_last = len(limits)-1    \n",
    "        if  j > limits[i_last]:\n",
    "            suf = str(limits[i_last])\n",
    "            break\n",
    "    return suf      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Train the Agent  with Vectorized Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "num_updates=1000000\n",
    "gamma = 0.99\n",
    "tau=0.95\n",
    "save_interval=30\n",
    "log_interval= 1 \n",
    "\n",
    "def ppo_vec_env_train(envs, agent, policy, num_processes, num_steps, rollouts):\n",
    "    \n",
    "    time_start = time.time()\n",
    "    \n",
    "    n=len(envs.ps)    \n",
    "    envs.reset()\n",
    "    \n",
    "    # start all parallel agents\n",
    "    print('Number of agents: ', n)\n",
    "    envs.step([[1]*4]*n)\n",
    "    \n",
    "    indices = []\n",
    "    for i  in range(n):\n",
    "        indices.append(i)\n",
    "     \n",
    "    s = 0\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = []    \n",
    "\n",
    "    for i_episode in range(num_updates):\n",
    "        \n",
    "        total_reward = np.zeros(n)\n",
    "        timestep = 0\n",
    "        \n",
    "        for timestep in range(num_steps):\n",
    "\n",
    "            with torch.no_grad():\n",
    "                value, actions, action_log_prob, recurrent_hidden_states = \\\n",
    "                   policy.act(\n",
    "                        rollouts.obs[timestep],\n",
    "                        rollouts.recurrent_hidden_states[timestep],\n",
    "                        rollouts.masks[timestep])\n",
    "                \n",
    "            obs, rewards, done, infos = envs.step(actions.cpu().detach().numpy())\n",
    "            \n",
    "            total_reward += rewards  ## this is the list by agents\n",
    "                        \n",
    "            # If done then clean the history of observations.\n",
    "            masks = torch.FloatTensor([[0.0] if done_ else [1.0] for done_ in done])\n",
    "            obs_t = torch.tensor(obs)\n",
    "            ## Add one dimnesion to tensor, otherwise does not work\n",
    "            ## This is (unsqueeze(1)) solution for:\n",
    "            ## RuntimeError: The expanded size of the tensor (1) must match the existing size...\n",
    "            rewards_t = torch.tensor(rewards).unsqueeze(1)\n",
    "            rollouts.insert(obs_t, recurrent_hidden_states, actions, action_log_prob, \\\n",
    "                value, rewards_t, masks)\n",
    "                    \n",
    "        avg_total_reward = np.mean(total_reward)\n",
    "        scores_deque.append(avg_total_reward)\n",
    "        scores_array.append(avg_total_reward)\n",
    "                \n",
    "        with torch.no_grad():\n",
    "            next_value = policy.get_value(rollouts.obs[-1],\n",
    "                            rollouts.recurrent_hidden_states[-1],\n",
    "                            rollouts.masks[-1]).detach()\n",
    "\n",
    "        rollouts.compute_returns(next_value, gamma, tau)\n",
    "\n",
    "        agent.update(rollouts)\n",
    "\n",
    "        rollouts.after_update()\n",
    "        \n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "\n",
    "        if i_episode > 0 and i_episode % save_interval == 0:\n",
    "            print('Saving model, i_episode: ', i_episode, '\\n')\n",
    "            suf = return_suffix(avg_score)\n",
    "            save(policy, 'dir_save', 'we0', suf)\n",
    "            #save_venv(policy, 'dir_save_VecEnv', 'final')\n",
    "\n",
    "        \n",
    "        if i_episode % log_interval == 0 and len(scores_deque) > 1:            \n",
    "            prev_s = s\n",
    "            s = (int)(time.time() - time_start)\n",
    "            t_del = s - prev_s\n",
    "            print('Ep. {}, Timesteps {}, Score.Agents: {:.2f}, Avg.Score: {:.2f}, Time: {:02}:{:02}:{:02}, \\\n",
    "Interval: {:02}:{:02}'\\\n",
    "                   .format(i_episode, timestep+1, \\\n",
    "                        avg_total_reward, avg_score, s//3600, s%3600//60, s%60, t_del%3600//60, t_del%60)) \n",
    "    \n",
    "        if len(scores_deque) == 100 and np.mean(scores_deque) > 300.5:   \n",
    "        # if np.mean(scores_deque) > 20:   \n",
    "            print('Environment solved with Average Score: ',  np.mean(scores_deque) )\n",
    "            break\n",
    "    \n",
    "    \n",
    "    return scores_array, avg_scores_array\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents:  16\n",
      "Ep. 1, Timesteps 1600, Score.Agents: -621.02, Avg.Score: -967.44, Time: 00:00:48, Interval: 00:48\n",
      "Ep. 2, Timesteps 1600, Score.Agents: -618.29, Avg.Score: -851.06, Time: 00:01:11, Interval: 00:23\n",
      "Ep. 3, Timesteps 1600, Score.Agents: -530.05, Avg.Score: -770.80, Time: 00:01:33, Interval: 00:22\n",
      "Ep. 4, Timesteps 1600, Score.Agents: -316.62, Avg.Score: -679.97, Time: 00:01:54, Interval: 00:21\n",
      "Ep. 5, Timesteps 1600, Score.Agents: -256.20, Avg.Score: -609.34, Time: 00:02:14, Interval: 00:20\n",
      "Ep. 6, Timesteps 1600, Score.Agents: -274.00, Avg.Score: -561.43, Time: 00:02:35, Interval: 00:21\n",
      "Ep. 7, Timesteps 1600, Score.Agents: -238.21, Avg.Score: -521.03, Time: 00:02:56, Interval: 00:21\n",
      "Ep. 8, Timesteps 1600, Score.Agents: -207.62, Avg.Score: -486.21, Time: 00:03:17, Interval: 00:21\n",
      "Ep. 9, Timesteps 1600, Score.Agents: -175.54, Avg.Score: -455.14, Time: 00:03:37, Interval: 00:20\n",
      "Ep. 10, Timesteps 1600, Score.Agents: -147.44, Avg.Score: -427.17, Time: 00:03:58, Interval: 00:21\n",
      "Ep. 11, Timesteps 1600, Score.Agents: -209.80, Avg.Score: -409.05, Time: 00:04:18, Interval: 00:20\n",
      "Ep. 12, Timesteps 1600, Score.Agents: -157.99, Avg.Score: -389.74, Time: 00:04:39, Interval: 00:21\n",
      "Ep. 13, Timesteps 1600, Score.Agents: -170.08, Avg.Score: -374.05, Time: 00:04:59, Interval: 00:20\n",
      "Ep. 14, Timesteps 1600, Score.Agents: -159.83, Avg.Score: -359.77, Time: 00:05:20, Interval: 00:21\n",
      "Ep. 15, Timesteps 1600, Score.Agents: -144.27, Avg.Score: -346.30, Time: 00:05:41, Interval: 00:21\n",
      "Ep. 16, Timesteps 1600, Score.Agents: -140.29, Avg.Score: -334.18, Time: 00:06:01, Interval: 00:20\n",
      "Ep. 17, Timesteps 1600, Score.Agents: -142.43, Avg.Score: -323.53, Time: 00:06:22, Interval: 00:21\n",
      "Ep. 18, Timesteps 1600, Score.Agents: -148.50, Avg.Score: -314.32, Time: 00:06:42, Interval: 00:20\n",
      "Ep. 19, Timesteps 1600, Score.Agents: -156.60, Avg.Score: -306.43, Time: 00:07:03, Interval: 00:21\n",
      "Ep. 20, Timesteps 1600, Score.Agents: -168.49, Avg.Score: -299.86, Time: 00:07:23, Interval: 00:20\n",
      "Ep. 21, Timesteps 1600, Score.Agents: -135.48, Avg.Score: -292.39, Time: 00:07:44, Interval: 00:21\n",
      "Ep. 22, Timesteps 1600, Score.Agents: -145.45, Avg.Score: -286.00, Time: 00:08:04, Interval: 00:20\n",
      "Ep. 23, Timesteps 1600, Score.Agents: -119.91, Avg.Score: -279.08, Time: 00:08:25, Interval: 00:21\n",
      "Ep. 24, Timesteps 1600, Score.Agents: -141.48, Avg.Score: -273.58, Time: 00:08:45, Interval: 00:20\n",
      "Ep. 25, Timesteps 1600, Score.Agents: -163.07, Avg.Score: -269.33, Time: 00:09:06, Interval: 00:21\n",
      "Ep. 26, Timesteps 1600, Score.Agents: -161.63, Avg.Score: -265.34, Time: 00:09:26, Interval: 00:20\n",
      "Ep. 27, Timesteps 1600, Score.Agents: -134.63, Avg.Score: -260.67, Time: 00:09:47, Interval: 00:21\n",
      "Ep. 28, Timesteps 1600, Score.Agents: -131.02, Avg.Score: -256.20, Time: 00:10:07, Interval: 00:20\n",
      "Ep. 29, Timesteps 1600, Score.Agents: -131.32, Avg.Score: -252.04, Time: 00:10:28, Interval: 00:21\n",
      "Saving model, i_episode:  30 \n",
      "\n",
      "Ep. 30, Timesteps 1600, Score.Agents: -131.76, Avg.Score: -248.16, Time: 00:10:48, Interval: 00:20\n",
      "Ep. 31, Timesteps 1600, Score.Agents: -128.53, Avg.Score: -244.42, Time: 00:11:09, Interval: 00:21\n",
      "Ep. 32, Timesteps 1600, Score.Agents: -116.76, Avg.Score: -240.55, Time: 00:11:30, Interval: 00:21\n",
      "Ep. 33, Timesteps 1600, Score.Agents: -129.28, Avg.Score: -237.28, Time: 00:11:50, Interval: 00:20\n",
      "Ep. 34, Timesteps 1600, Score.Agents: -127.34, Avg.Score: -234.14, Time: 00:12:11, Interval: 00:21\n",
      "Ep. 35, Timesteps 1600, Score.Agents: -122.66, Avg.Score: -231.04, Time: 00:12:32, Interval: 00:21\n",
      "Ep. 36, Timesteps 1600, Score.Agents: -108.43, Avg.Score: -227.73, Time: 00:12:52, Interval: 00:20\n",
      "Ep. 37, Timesteps 1600, Score.Agents: -119.59, Avg.Score: -224.88, Time: 00:13:12, Interval: 00:20\n",
      "Ep. 38, Timesteps 1600, Score.Agents: -107.36, Avg.Score: -221.87, Time: 00:13:33, Interval: 00:21\n",
      "Ep. 39, Timesteps 1600, Score.Agents: -109.58, Avg.Score: -219.06, Time: 00:13:53, Interval: 00:20\n",
      "Ep. 40, Timesteps 1600, Score.Agents: -93.78, Avg.Score: -216.00, Time: 00:14:14, Interval: 00:21\n",
      "Ep. 41, Timesteps 1600, Score.Agents: -117.19, Avg.Score: -213.65, Time: 00:14:34, Interval: 00:20\n",
      "Ep. 42, Timesteps 1600, Score.Agents: -93.15, Avg.Score: -210.85, Time: 00:14:55, Interval: 00:21\n",
      "Ep. 43, Timesteps 1600, Score.Agents: -88.95, Avg.Score: -208.08, Time: 00:15:15, Interval: 00:20\n",
      "Ep. 44, Timesteps 1600, Score.Agents: -104.96, Avg.Score: -205.79, Time: 00:15:35, Interval: 00:20\n",
      "Ep. 45, Timesteps 1600, Score.Agents: -87.98, Avg.Score: -203.23, Time: 00:15:56, Interval: 00:21\n",
      "Ep. 46, Timesteps 1600, Score.Agents: -85.46, Avg.Score: -200.72, Time: 00:16:16, Interval: 00:20\n",
      "Ep. 47, Timesteps 1600, Score.Agents: -87.72, Avg.Score: -198.37, Time: 00:16:36, Interval: 00:20\n",
      "Ep. 48, Timesteps 1600, Score.Agents: -77.51, Avg.Score: -195.90, Time: 00:16:57, Interval: 00:21\n",
      "Ep. 49, Timesteps 1600, Score.Agents: -74.26, Avg.Score: -193.47, Time: 00:17:17, Interval: 00:20\n",
      "Ep. 50, Timesteps 1600, Score.Agents: -72.96, Avg.Score: -191.10, Time: 00:17:38, Interval: 00:21\n",
      "Ep. 51, Timesteps 1600, Score.Agents: -71.67, Avg.Score: -188.81, Time: 00:17:58, Interval: 00:20\n",
      "Ep. 52, Timesteps 1600, Score.Agents: -71.41, Avg.Score: -186.59, Time: 00:18:18, Interval: 00:20\n",
      "Ep. 53, Timesteps 1600, Score.Agents: -64.29, Avg.Score: -184.33, Time: 00:18:39, Interval: 00:21\n",
      "Ep. 54, Timesteps 1600, Score.Agents: -56.40, Avg.Score: -182.00, Time: 00:18:59, Interval: 00:20\n",
      "Ep. 55, Timesteps 1600, Score.Agents: -57.03, Avg.Score: -179.77, Time: 00:19:19, Interval: 00:20\n",
      "Ep. 56, Timesteps 1600, Score.Agents: -60.42, Avg.Score: -177.68, Time: 00:19:40, Interval: 00:21\n",
      "Ep. 57, Timesteps 1600, Score.Agents: -44.69, Avg.Score: -175.38, Time: 00:20:00, Interval: 00:20\n",
      "Ep. 58, Timesteps 1600, Score.Agents: -33.97, Avg.Score: -172.99, Time: 00:20:21, Interval: 00:21\n",
      "Ep. 59, Timesteps 1600, Score.Agents: -44.59, Avg.Score: -170.85, Time: 00:20:41, Interval: 00:20\n",
      "Saving model, i_episode:  60 \n",
      "\n",
      "Ep. 60, Timesteps 1600, Score.Agents: -37.38, Avg.Score: -168.66, Time: 00:21:01, Interval: 00:20\n",
      "Ep. 61, Timesteps 1600, Score.Agents: -27.09, Avg.Score: -166.38, Time: 00:21:21, Interval: 00:20\n",
      "Ep. 62, Timesteps 1600, Score.Agents: -28.02, Avg.Score: -164.18, Time: 00:21:42, Interval: 00:21\n",
      "Ep. 63, Timesteps 1600, Score.Agents: -11.83, Avg.Score: -161.80, Time: 00:22:02, Interval: 00:20\n",
      "Ep. 64, Timesteps 1600, Score.Agents: -0.05, Avg.Score: -159.31, Time: 00:22:22, Interval: 00:20\n",
      "Ep. 65, Timesteps 1600, Score.Agents: 6.91, Avg.Score: -156.79, Time: 00:22:42, Interval: 00:20\n",
      "Ep. 66, Timesteps 1600, Score.Agents: 0.46, Avg.Score: -154.44, Time: 00:23:02, Interval: 00:20\n",
      "Ep. 67, Timesteps 1600, Score.Agents: 14.99, Avg.Score: -151.95, Time: 00:23:23, Interval: 00:21\n",
      "Ep. 68, Timesteps 1600, Score.Agents: 25.19, Avg.Score: -149.39, Time: 00:23:43, Interval: 00:20\n",
      "Ep. 69, Timesteps 1600, Score.Agents: 11.61, Avg.Score: -147.09, Time: 00:24:03, Interval: 00:20\n",
      "Ep. 70, Timesteps 1600, Score.Agents: 21.14, Avg.Score: -144.72, Time: 00:24:24, Interval: 00:21\n",
      "Ep. 71, Timesteps 1600, Score.Agents: 34.62, Avg.Score: -142.23, Time: 00:24:44, Interval: 00:20\n",
      "Ep. 72, Timesteps 1600, Score.Agents: 30.32, Avg.Score: -139.86, Time: 00:25:04, Interval: 00:20\n",
      "Ep. 73, Timesteps 1600, Score.Agents: 50.55, Avg.Score: -137.29, Time: 00:25:24, Interval: 00:20\n",
      "Ep. 74, Timesteps 1600, Score.Agents: 61.95, Avg.Score: -134.63, Time: 00:25:45, Interval: 00:21\n",
      "Ep. 75, Timesteps 1600, Score.Agents: 54.19, Avg.Score: -132.15, Time: 00:26:05, Interval: 00:20\n",
      "Ep. 76, Timesteps 1600, Score.Agents: 58.53, Avg.Score: -129.67, Time: 00:26:26, Interval: 00:21\n",
      "Ep. 77, Timesteps 1600, Score.Agents: 68.90, Avg.Score: -127.13, Time: 00:26:46, Interval: 00:20\n",
      "Ep. 78, Timesteps 1600, Score.Agents: 68.17, Avg.Score: -124.65, Time: 00:27:06, Interval: 00:20\n",
      "Ep. 79, Timesteps 1600, Score.Agents: 59.01, Avg.Score: -122.36, Time: 00:27:27, Interval: 00:21\n",
      "Ep. 80, Timesteps 1600, Score.Agents: 62.07, Avg.Score: -120.08, Time: 00:27:47, Interval: 00:20\n",
      "Ep. 81, Timesteps 1600, Score.Agents: 52.96, Avg.Score: -117.97, Time: 00:28:07, Interval: 00:20\n",
      "Ep. 82, Timesteps 1600, Score.Agents: 77.72, Avg.Score: -115.61, Time: 00:28:27, Interval: 00:20\n",
      "Ep. 83, Timesteps 1600, Score.Agents: 62.40, Avg.Score: -113.49, Time: 00:28:48, Interval: 00:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 84, Timesteps 1600, Score.Agents: 64.69, Avg.Score: -111.40, Time: 00:29:08, Interval: 00:20\n",
      "Ep. 85, Timesteps 1600, Score.Agents: 58.87, Avg.Score: -109.42, Time: 00:29:29, Interval: 00:21\n",
      "Ep. 86, Timesteps 1600, Score.Agents: 49.08, Avg.Score: -107.60, Time: 00:29:49, Interval: 00:20\n",
      "Ep. 87, Timesteps 1600, Score.Agents: 58.61, Avg.Score: -105.71, Time: 00:30:09, Interval: 00:20\n",
      "Ep. 88, Timesteps 1600, Score.Agents: 66.82, Avg.Score: -103.77, Time: 00:30:30, Interval: 00:21\n",
      "Ep. 89, Timesteps 1600, Score.Agents: 79.53, Avg.Score: -101.73, Time: 00:30:50, Interval: 00:20\n",
      "Saving model, i_episode:  90 \n",
      "\n",
      "Ep. 90, Timesteps 1600, Score.Agents: 65.72, Avg.Score: -99.89, Time: 00:31:10, Interval: 00:20\n",
      "Ep. 91, Timesteps 1600, Score.Agents: 85.11, Avg.Score: -97.88, Time: 00:31:30, Interval: 00:20\n",
      "Ep. 92, Timesteps 1600, Score.Agents: 66.34, Avg.Score: -96.11, Time: 00:31:51, Interval: 00:21\n",
      "Ep. 93, Timesteps 1600, Score.Agents: 77.33, Avg.Score: -94.27, Time: 00:32:11, Interval: 00:20\n",
      "Ep. 94, Timesteps 1600, Score.Agents: 97.46, Avg.Score: -92.25, Time: 00:32:31, Interval: 00:20\n",
      "Ep. 95, Timesteps 1600, Score.Agents: 95.79, Avg.Score: -90.29, Time: 00:32:51, Interval: 00:20\n",
      "Ep. 96, Timesteps 1600, Score.Agents: 87.15, Avg.Score: -88.46, Time: 00:33:12, Interval: 00:21\n",
      "Ep. 97, Timesteps 1600, Score.Agents: 90.32, Avg.Score: -86.64, Time: 00:33:32, Interval: 00:20\n",
      "Ep. 98, Timesteps 1600, Score.Agents: 105.13, Avg.Score: -84.70, Time: 00:33:52, Interval: 00:20\n",
      "Ep. 99, Timesteps 1600, Score.Agents: 104.48, Avg.Score: -82.81, Time: 00:34:12, Interval: 00:20\n",
      "Ep. 100, Timesteps 1600, Score.Agents: 92.82, Avg.Score: -68.74, Time: 00:34:32, Interval: 00:20\n",
      "Ep. 101, Timesteps 1600, Score.Agents: 97.50, Avg.Score: -61.56, Time: 00:34:53, Interval: 00:21\n",
      "Ep. 102, Timesteps 1600, Score.Agents: 100.94, Avg.Score: -54.37, Time: 00:35:13, Interval: 00:20\n",
      "Ep. 103, Timesteps 1600, Score.Agents: 106.46, Avg.Score: -48.00, Time: 00:35:33, Interval: 00:20\n",
      "Ep. 104, Timesteps 1600, Score.Agents: 93.37, Avg.Score: -43.90, Time: 00:35:53, Interval: 00:20\n",
      "Ep. 105, Timesteps 1600, Score.Agents: 96.52, Avg.Score: -40.37, Time: 00:36:14, Interval: 00:21\n",
      "Ep. 106, Timesteps 1600, Score.Agents: 86.43, Avg.Score: -36.77, Time: 00:36:34, Interval: 00:20\n",
      "Ep. 107, Timesteps 1600, Score.Agents: 105.04, Avg.Score: -33.34, Time: 00:36:54, Interval: 00:20\n",
      "Ep. 108, Timesteps 1600, Score.Agents: 115.79, Avg.Score: -30.10, Time: 00:37:14, Interval: 00:20\n",
      "Ep. 109, Timesteps 1600, Score.Agents: 119.74, Avg.Score: -27.15, Time: 00:37:35, Interval: 00:21\n",
      "Ep. 110, Timesteps 1600, Score.Agents: 99.25, Avg.Score: -24.68, Time: 00:37:55, Interval: 00:20\n",
      "Ep. 111, Timesteps 1600, Score.Agents: 118.52, Avg.Score: -21.40, Time: 00:38:15, Interval: 00:20\n",
      "Ep. 112, Timesteps 1600, Score.Agents: 103.78, Avg.Score: -18.78, Time: 00:38:36, Interval: 00:21\n",
      "Ep. 113, Timesteps 1600, Score.Agents: 117.74, Avg.Score: -15.90, Time: 00:38:56, Interval: 00:20\n",
      "Ep. 114, Timesteps 1600, Score.Agents: 111.14, Avg.Score: -13.19, Time: 00:39:16, Interval: 00:20\n",
      "Ep. 115, Timesteps 1600, Score.Agents: 117.81, Avg.Score: -10.57, Time: 00:39:36, Interval: 00:20\n",
      "Ep. 116, Timesteps 1600, Score.Agents: 121.75, Avg.Score: -7.95, Time: 00:39:56, Interval: 00:20\n",
      "Ep. 117, Timesteps 1600, Score.Agents: 113.89, Avg.Score: -5.39, Time: 00:40:17, Interval: 00:21\n",
      "Ep. 118, Timesteps 1600, Score.Agents: 123.49, Avg.Score: -2.67, Time: 00:40:37, Interval: 00:20\n",
      "Ep. 119, Timesteps 1600, Score.Agents: 130.62, Avg.Score: 0.20, Time: 00:40:57, Interval: 00:20\n",
      "Saving model, i_episode:  120 \n",
      "\n",
      "Ep. 120, Timesteps 1600, Score.Agents: 144.18, Avg.Score: 3.33, Time: 00:41:17, Interval: 00:20\n",
      "Ep. 121, Timesteps 1600, Score.Agents: 137.09, Avg.Score: 6.05, Time: 00:41:38, Interval: 00:21\n",
      "Ep. 122, Timesteps 1600, Score.Agents: 135.07, Avg.Score: 8.86, Time: 00:41:58, Interval: 00:20\n",
      "Ep. 123, Timesteps 1600, Score.Agents: 145.32, Avg.Score: 11.51, Time: 00:42:18, Interval: 00:20\n",
      "Ep. 124, Timesteps 1600, Score.Agents: 152.28, Avg.Score: 14.45, Time: 00:42:38, Interval: 00:20\n",
      "Ep. 125, Timesteps 1600, Score.Agents: 153.74, Avg.Score: 17.62, Time: 00:42:58, Interval: 00:20\n",
      "Ep. 126, Timesteps 1600, Score.Agents: 139.73, Avg.Score: 20.63, Time: 00:43:19, Interval: 00:21\n",
      "Ep. 127, Timesteps 1600, Score.Agents: 148.36, Avg.Score: 23.46, Time: 00:43:39, Interval: 00:20\n",
      "Ep. 128, Timesteps 1600, Score.Agents: 137.75, Avg.Score: 26.15, Time: 00:43:59, Interval: 00:20\n",
      "Ep. 129, Timesteps 1600, Score.Agents: 134.84, Avg.Score: 28.81, Time: 00:44:19, Interval: 00:20\n",
      "Ep. 130, Timesteps 1600, Score.Agents: 137.62, Avg.Score: 31.50, Time: 00:44:40, Interval: 00:21\n",
      "Ep. 131, Timesteps 1600, Score.Agents: 148.49, Avg.Score: 34.27, Time: 00:45:00, Interval: 00:20\n",
      "Ep. 132, Timesteps 1600, Score.Agents: 151.78, Avg.Score: 36.96, Time: 00:45:20, Interval: 00:20\n",
      "Ep. 133, Timesteps 1600, Score.Agents: 149.31, Avg.Score: 39.75, Time: 00:45:40, Interval: 00:20\n",
      "Ep. 134, Timesteps 1600, Score.Agents: 145.30, Avg.Score: 42.47, Time: 00:46:00, Interval: 00:20\n",
      "Ep. 135, Timesteps 1600, Score.Agents: 154.72, Avg.Score: 45.25, Time: 00:46:20, Interval: 00:20\n",
      "Ep. 136, Timesteps 1600, Score.Agents: 153.87, Avg.Score: 47.87, Time: 00:46:41, Interval: 00:21\n",
      "Ep. 137, Timesteps 1600, Score.Agents: 154.44, Avg.Score: 50.61, Time: 00:47:01, Interval: 00:20\n",
      "Ep. 138, Timesteps 1600, Score.Agents: 164.96, Avg.Score: 53.33, Time: 00:47:21, Interval: 00:20\n",
      "Ep. 139, Timesteps 1600, Score.Agents: 179.25, Avg.Score: 56.22, Time: 00:47:41, Interval: 00:20\n",
      "Ep. 140, Timesteps 1600, Score.Agents: 158.86, Avg.Score: 58.75, Time: 00:48:01, Interval: 00:20\n",
      "Ep. 141, Timesteps 1600, Score.Agents: 167.98, Avg.Score: 61.60, Time: 00:48:21, Interval: 00:20\n",
      "Ep. 142, Timesteps 1600, Score.Agents: 165.15, Avg.Score: 64.18, Time: 00:48:41, Interval: 00:20\n",
      "Ep. 143, Timesteps 1600, Score.Agents: 164.40, Avg.Score: 66.72, Time: 00:49:02, Interval: 00:21\n",
      "Ep. 144, Timesteps 1600, Score.Agents: 167.08, Avg.Score: 69.44, Time: 00:49:22, Interval: 00:20\n",
      "Ep. 145, Timesteps 1600, Score.Agents: 154.11, Avg.Score: 71.86, Time: 00:49:43, Interval: 00:21\n",
      "Ep. 146, Timesteps 1600, Score.Agents: 184.19, Avg.Score: 74.55, Time: 00:50:03, Interval: 00:20\n",
      "Ep. 147, Timesteps 1600, Score.Agents: 189.92, Avg.Score: 77.33, Time: 00:50:24, Interval: 00:21\n",
      "Ep. 148, Timesteps 1600, Score.Agents: 181.03, Avg.Score: 79.91, Time: 00:50:44, Interval: 00:20\n",
      "Ep. 149, Timesteps 1600, Score.Agents: 183.81, Avg.Score: 82.50, Time: 00:51:05, Interval: 00:21\n",
      "Saving model, i_episode:  150 \n",
      "\n",
      "Ep. 150, Timesteps 1600, Score.Agents: 178.08, Avg.Score: 85.01, Time: 00:51:25, Interval: 00:20\n",
      "Ep. 151, Timesteps 1600, Score.Agents: 176.11, Avg.Score: 87.48, Time: 00:51:45, Interval: 00:20\n",
      "Ep. 152, Timesteps 1600, Score.Agents: 176.72, Avg.Score: 89.96, Time: 00:52:05, Interval: 00:20\n",
      "Ep. 153, Timesteps 1600, Score.Agents: 182.07, Avg.Score: 92.43, Time: 00:52:25, Interval: 00:20\n",
      "Ep. 154, Timesteps 1600, Score.Agents: 196.43, Avg.Score: 94.96, Time: 00:52:45, Interval: 00:20\n",
      "Ep. 155, Timesteps 1600, Score.Agents: 186.61, Avg.Score: 97.39, Time: 00:53:05, Interval: 00:20\n",
      "Ep. 156, Timesteps 1600, Score.Agents: 190.40, Avg.Score: 99.90, Time: 00:53:26, Interval: 00:21\n",
      "Ep. 157, Timesteps 1600, Score.Agents: 201.86, Avg.Score: 102.37, Time: 00:53:46, Interval: 00:20\n",
      "Ep. 158, Timesteps 1600, Score.Agents: 197.35, Avg.Score: 104.68, Time: 00:54:06, Interval: 00:20\n",
      "Ep. 159, Timesteps 1600, Score.Agents: 203.24, Avg.Score: 107.16, Time: 00:54:26, Interval: 00:20\n",
      "Ep. 160, Timesteps 1600, Score.Agents: 201.38, Avg.Score: 109.55, Time: 00:54:47, Interval: 00:21\n",
      "Ep. 161, Timesteps 1600, Score.Agents: 172.01, Avg.Score: 111.54, Time: 00:55:07, Interval: 00:20\n",
      "Ep. 162, Timesteps 1600, Score.Agents: 179.95, Avg.Score: 113.62, Time: 00:55:28, Interval: 00:21\n",
      "Ep. 163, Timesteps 1600, Score.Agents: 202.46, Avg.Score: 115.76, Time: 00:55:48, Interval: 00:20\n",
      "Ep. 164, Timesteps 1600, Score.Agents: 194.47, Avg.Score: 117.70, Time: 00:56:08, Interval: 00:20\n",
      "Ep. 165, Timesteps 1600, Score.Agents: 196.05, Avg.Score: 119.60, Time: 00:56:28, Interval: 00:20\n",
      "Ep. 166, Timesteps 1600, Score.Agents: 210.96, Avg.Score: 121.70, Time: 00:56:48, Interval: 00:20\n",
      "Ep. 167, Timesteps 1600, Score.Agents: 198.73, Avg.Score: 123.54, Time: 00:57:08, Interval: 00:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 168, Timesteps 1600, Score.Agents: 195.10, Avg.Score: 125.24, Time: 00:57:29, Interval: 00:21\n",
      "Ep. 169, Timesteps 1600, Score.Agents: 198.95, Avg.Score: 127.11, Time: 00:57:49, Interval: 00:20\n",
      "Ep. 170, Timesteps 1600, Score.Agents: 220.08, Avg.Score: 129.10, Time: 00:58:09, Interval: 00:20\n",
      "Ep. 171, Timesteps 1600, Score.Agents: 213.32, Avg.Score: 130.89, Time: 00:58:30, Interval: 00:21\n",
      "Ep. 172, Timesteps 1600, Score.Agents: 222.26, Avg.Score: 132.81, Time: 00:58:50, Interval: 00:20\n",
      "Ep. 173, Timesteps 1600, Score.Agents: 208.55, Avg.Score: 134.39, Time: 00:59:11, Interval: 00:21\n",
      "Ep. 174, Timesteps 1600, Score.Agents: 219.82, Avg.Score: 135.97, Time: 00:59:32, Interval: 00:21\n",
      "Ep. 175, Timesteps 1600, Score.Agents: 219.89, Avg.Score: 137.62, Time: 00:59:52, Interval: 00:20\n",
      "Ep. 176, Timesteps 1600, Score.Agents: 211.71, Avg.Score: 139.15, Time: 01:00:12, Interval: 00:20\n",
      "Ep. 177, Timesteps 1600, Score.Agents: 223.40, Avg.Score: 140.70, Time: 01:00:32, Interval: 00:20\n",
      "Ep. 178, Timesteps 1600, Score.Agents: 215.42, Avg.Score: 142.17, Time: 01:00:53, Interval: 00:21\n",
      "Ep. 179, Timesteps 1600, Score.Agents: 223.44, Avg.Score: 143.82, Time: 01:01:13, Interval: 00:20\n",
      "Saving model, i_episode:  180 \n",
      "\n",
      "Ep. 180, Timesteps 1600, Score.Agents: 203.09, Avg.Score: 145.23, Time: 01:01:33, Interval: 00:20\n",
      "Ep. 181, Timesteps 1600, Score.Agents: 226.15, Avg.Score: 146.96, Time: 01:01:53, Interval: 00:20\n",
      "Ep. 182, Timesteps 1600, Score.Agents: 221.31, Avg.Score: 148.39, Time: 01:02:13, Interval: 00:20\n",
      "Ep. 183, Timesteps 1600, Score.Agents: 233.07, Avg.Score: 150.10, Time: 01:02:33, Interval: 00:20\n",
      "Ep. 184, Timesteps 1600, Score.Agents: 194.74, Avg.Score: 151.40, Time: 01:02:53, Interval: 00:20\n",
      "Ep. 185, Timesteps 1600, Score.Agents: 230.64, Avg.Score: 153.12, Time: 01:03:14, Interval: 00:21\n",
      "Ep. 186, Timesteps 1600, Score.Agents: 234.93, Avg.Score: 154.98, Time: 01:03:34, Interval: 00:20\n",
      "Ep. 187, Timesteps 1600, Score.Agents: 235.26, Avg.Score: 156.74, Time: 01:03:54, Interval: 00:20\n",
      "Ep. 188, Timesteps 1600, Score.Agents: 240.20, Avg.Score: 158.48, Time: 01:04:15, Interval: 00:21\n",
      "Ep. 189, Timesteps 1600, Score.Agents: 233.28, Avg.Score: 160.02, Time: 01:04:35, Interval: 00:20\n",
      "Ep. 190, Timesteps 1600, Score.Agents: 220.80, Avg.Score: 161.57, Time: 01:04:56, Interval: 00:21\n",
      "Ep. 191, Timesteps 1600, Score.Agents: 226.76, Avg.Score: 162.98, Time: 01:05:16, Interval: 00:20\n",
      "Ep. 192, Timesteps 1600, Score.Agents: 234.88, Avg.Score: 164.67, Time: 01:05:36, Interval: 00:20\n",
      "Ep. 193, Timesteps 1600, Score.Agents: 240.15, Avg.Score: 166.30, Time: 01:05:56, Interval: 00:20\n",
      "Ep. 194, Timesteps 1600, Score.Agents: 227.30, Avg.Score: 167.59, Time: 01:06:17, Interval: 00:21\n",
      "Ep. 195, Timesteps 1600, Score.Agents: 229.26, Avg.Score: 168.93, Time: 01:06:37, Interval: 00:20\n",
      "Ep. 196, Timesteps 1600, Score.Agents: 241.34, Avg.Score: 170.47, Time: 01:06:58, Interval: 00:21\n",
      "Ep. 197, Timesteps 1600, Score.Agents: 237.83, Avg.Score: 171.95, Time: 01:07:18, Interval: 00:20\n",
      "Ep. 198, Timesteps 1600, Score.Agents: 241.56, Avg.Score: 173.31, Time: 01:07:38, Interval: 00:20\n",
      "Ep. 199, Timesteps 1600, Score.Agents: 237.31, Avg.Score: 174.64, Time: 01:07:59, Interval: 00:21\n",
      "Ep. 200, Timesteps 1600, Score.Agents: 248.88, Avg.Score: 176.20, Time: 01:08:19, Interval: 00:20\n",
      "Ep. 201, Timesteps 1600, Score.Agents: 249.42, Avg.Score: 177.72, Time: 01:08:39, Interval: 00:20\n",
      "Ep. 202, Timesteps 1600, Score.Agents: 248.29, Avg.Score: 179.19, Time: 01:08:59, Interval: 00:20\n",
      "Ep. 203, Timesteps 1600, Score.Agents: 238.01, Avg.Score: 180.51, Time: 01:09:20, Interval: 00:21\n",
      "Ep. 204, Timesteps 1600, Score.Agents: 235.72, Avg.Score: 181.93, Time: 01:09:41, Interval: 00:21\n",
      "Ep. 205, Timesteps 1600, Score.Agents: 226.58, Avg.Score: 183.23, Time: 01:10:01, Interval: 00:20\n",
      "Ep. 206, Timesteps 1600, Score.Agents: 249.12, Avg.Score: 184.86, Time: 01:10:21, Interval: 00:20\n",
      "Ep. 207, Timesteps 1600, Score.Agents: 244.63, Avg.Score: 186.25, Time: 01:10:42, Interval: 00:21\n",
      "Ep. 208, Timesteps 1600, Score.Agents: 256.25, Avg.Score: 187.66, Time: 01:11:02, Interval: 00:20\n",
      "Ep. 209, Timesteps 1600, Score.Agents: 260.33, Avg.Score: 189.07, Time: 01:11:23, Interval: 00:21\n",
      "Saving model, i_episode:  210 \n",
      "\n",
      "Ep. 210, Timesteps 1600, Score.Agents: 235.35, Avg.Score: 190.43, Time: 01:11:43, Interval: 00:20\n",
      "Ep. 211, Timesteps 1600, Score.Agents: 234.63, Avg.Score: 191.59, Time: 01:12:03, Interval: 00:20\n",
      "Ep. 212, Timesteps 1600, Score.Agents: 246.61, Avg.Score: 193.02, Time: 01:12:24, Interval: 00:21\n",
      "Ep. 213, Timesteps 1600, Score.Agents: 243.95, Avg.Score: 194.28, Time: 01:12:44, Interval: 00:20\n",
      "Ep. 214, Timesteps 1600, Score.Agents: 244.30, Avg.Score: 195.61, Time: 01:13:04, Interval: 00:20\n",
      "Ep. 215, Timesteps 1600, Score.Agents: 242.58, Avg.Score: 196.86, Time: 01:13:24, Interval: 00:20\n",
      "Ep. 216, Timesteps 1600, Score.Agents: 256.09, Avg.Score: 198.20, Time: 01:13:45, Interval: 00:21\n",
      "Ep. 217, Timesteps 1600, Score.Agents: 263.72, Avg.Score: 199.70, Time: 01:14:05, Interval: 00:20\n",
      "Ep. 218, Timesteps 1600, Score.Agents: 251.11, Avg.Score: 200.97, Time: 01:14:26, Interval: 00:21\n",
      "Ep. 219, Timesteps 1600, Score.Agents: 251.06, Avg.Score: 202.18, Time: 01:14:47, Interval: 00:21\n",
      "Ep. 220, Timesteps 1600, Score.Agents: 253.61, Avg.Score: 203.27, Time: 01:15:08, Interval: 00:21\n",
      "Ep. 221, Timesteps 1600, Score.Agents: 260.77, Avg.Score: 204.51, Time: 01:15:29, Interval: 00:21\n",
      "Ep. 222, Timesteps 1600, Score.Agents: 259.99, Avg.Score: 205.76, Time: 01:15:49, Interval: 00:20\n",
      "Ep. 223, Timesteps 1600, Score.Agents: 262.32, Avg.Score: 206.93, Time: 01:16:10, Interval: 00:21\n",
      "Ep. 224, Timesteps 1600, Score.Agents: 247.28, Avg.Score: 207.88, Time: 01:16:30, Interval: 00:20\n",
      "Ep. 225, Timesteps 1600, Score.Agents: 258.41, Avg.Score: 208.93, Time: 01:16:50, Interval: 00:20\n",
      "Ep. 226, Timesteps 1600, Score.Agents: 264.14, Avg.Score: 210.17, Time: 01:17:11, Interval: 00:21\n",
      "Ep. 227, Timesteps 1600, Score.Agents: 260.47, Avg.Score: 211.29, Time: 01:17:31, Interval: 00:20\n",
      "Ep. 228, Timesteps 1600, Score.Agents: 259.38, Avg.Score: 212.51, Time: 01:17:52, Interval: 00:21\n",
      "Ep. 229, Timesteps 1600, Score.Agents: 240.52, Avg.Score: 213.56, Time: 01:18:12, Interval: 00:20\n",
      "Ep. 230, Timesteps 1600, Score.Agents: 248.55, Avg.Score: 214.67, Time: 01:18:32, Interval: 00:20\n",
      "Ep. 231, Timesteps 1600, Score.Agents: 246.13, Avg.Score: 215.65, Time: 01:18:53, Interval: 00:21\n",
      "Ep. 232, Timesteps 1600, Score.Agents: 255.33, Avg.Score: 216.69, Time: 01:19:13, Interval: 00:20\n",
      "Ep. 233, Timesteps 1600, Score.Agents: 261.88, Avg.Score: 217.81, Time: 01:19:33, Interval: 00:20\n",
      "Ep. 234, Timesteps 1600, Score.Agents: 264.26, Avg.Score: 219.00, Time: 01:19:54, Interval: 00:21\n",
      "Ep. 235, Timesteps 1600, Score.Agents: 269.09, Avg.Score: 220.14, Time: 01:20:14, Interval: 00:20\n",
      "Ep. 236, Timesteps 1600, Score.Agents: 270.01, Avg.Score: 221.31, Time: 01:20:34, Interval: 00:20\n",
      "Ep. 237, Timesteps 1600, Score.Agents: 272.46, Avg.Score: 222.49, Time: 01:20:55, Interval: 00:21\n",
      "Ep. 238, Timesteps 1600, Score.Agents: 250.03, Avg.Score: 223.34, Time: 01:21:15, Interval: 00:20\n",
      "Ep. 239, Timesteps 1600, Score.Agents: 253.62, Avg.Score: 224.08, Time: 01:21:36, Interval: 00:21\n",
      "Saving model, i_episode:  240 \n",
      "\n",
      "Ep. 240, Timesteps 1600, Score.Agents: 269.91, Avg.Score: 225.19, Time: 01:21:56, Interval: 00:20\n",
      "Ep. 241, Timesteps 1600, Score.Agents: 268.17, Avg.Score: 226.19, Time: 01:22:16, Interval: 00:20\n",
      "Ep. 242, Timesteps 1600, Score.Agents: 268.81, Avg.Score: 227.23, Time: 01:22:36, Interval: 00:20\n",
      "Ep. 243, Timesteps 1600, Score.Agents: 248.79, Avg.Score: 228.07, Time: 01:22:57, Interval: 00:21\n",
      "Ep. 244, Timesteps 1600, Score.Agents: 263.17, Avg.Score: 229.03, Time: 01:23:17, Interval: 00:20\n",
      "Ep. 245, Timesteps 1600, Score.Agents: 262.98, Avg.Score: 230.12, Time: 01:23:38, Interval: 00:21\n",
      "Ep. 246, Timesteps 1600, Score.Agents: 250.32, Avg.Score: 230.78, Time: 01:23:58, Interval: 00:20\n",
      "Ep. 247, Timesteps 1600, Score.Agents: 260.56, Avg.Score: 231.49, Time: 01:24:18, Interval: 00:20\n",
      "Ep. 248, Timesteps 1600, Score.Agents: 262.41, Avg.Score: 232.30, Time: 01:24:38, Interval: 00:20\n",
      "Ep. 249, Timesteps 1600, Score.Agents: 252.34, Avg.Score: 232.99, Time: 01:24:59, Interval: 00:21\n",
      "Ep. 250, Timesteps 1600, Score.Agents: 263.68, Avg.Score: 233.85, Time: 01:25:19, Interval: 00:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 251, Timesteps 1600, Score.Agents: 262.84, Avg.Score: 234.71, Time: 01:25:39, Interval: 00:20\n",
      "Ep. 252, Timesteps 1600, Score.Agents: 256.09, Avg.Score: 235.51, Time: 01:25:59, Interval: 00:20\n",
      "Ep. 253, Timesteps 1600, Score.Agents: 262.34, Avg.Score: 236.31, Time: 01:26:19, Interval: 00:20\n",
      "Ep. 254, Timesteps 1600, Score.Agents: 269.85, Avg.Score: 237.04, Time: 01:26:39, Interval: 00:20\n",
      "Ep. 255, Timesteps 1600, Score.Agents: 278.74, Avg.Score: 237.97, Time: 01:27:00, Interval: 00:21\n",
      "Ep. 256, Timesteps 1600, Score.Agents: 269.22, Avg.Score: 238.75, Time: 01:27:21, Interval: 00:21\n",
      "Ep. 257, Timesteps 1600, Score.Agents: 276.96, Avg.Score: 239.50, Time: 01:27:41, Interval: 00:20\n",
      "Ep. 258, Timesteps 1600, Score.Agents: 275.01, Avg.Score: 240.28, Time: 01:28:01, Interval: 00:20\n",
      "Ep. 259, Timesteps 1600, Score.Agents: 260.99, Avg.Score: 240.86, Time: 01:28:22, Interval: 00:21\n",
      "Ep. 260, Timesteps 1600, Score.Agents: 259.19, Avg.Score: 241.44, Time: 01:28:42, Interval: 00:20\n",
      "Ep. 261, Timesteps 1600, Score.Agents: 271.14, Avg.Score: 242.43, Time: 01:29:02, Interval: 00:20\n",
      "Ep. 262, Timesteps 1600, Score.Agents: 278.51, Avg.Score: 243.41, Time: 01:29:23, Interval: 00:21\n",
      "Ep. 263, Timesteps 1600, Score.Agents: 281.21, Avg.Score: 244.20, Time: 01:29:43, Interval: 00:20\n",
      "Ep. 264, Timesteps 1600, Score.Agents: 262.44, Avg.Score: 244.88, Time: 01:30:04, Interval: 00:21\n",
      "Ep. 265, Timesteps 1600, Score.Agents: 287.74, Avg.Score: 245.80, Time: 01:30:24, Interval: 00:20\n",
      "Ep. 266, Timesteps 1600, Score.Agents: 278.46, Avg.Score: 246.47, Time: 01:30:45, Interval: 00:21\n",
      "Ep. 267, Timesteps 1600, Score.Agents: 278.15, Avg.Score: 247.27, Time: 01:31:06, Interval: 00:21\n",
      "Ep. 268, Timesteps 1600, Score.Agents: 284.09, Avg.Score: 248.16, Time: 01:31:26, Interval: 00:20\n",
      "Ep. 269, Timesteps 1600, Score.Agents: 252.66, Avg.Score: 248.69, Time: 01:31:47, Interval: 00:21\n",
      "Saving model, i_episode:  270 \n",
      "\n",
      "Ep. 270, Timesteps 1600, Score.Agents: 259.30, Avg.Score: 249.09, Time: 01:32:07, Interval: 00:20\n",
      "Ep. 271, Timesteps 1600, Score.Agents: 266.17, Avg.Score: 249.61, Time: 01:32:27, Interval: 00:20\n",
      "Ep. 272, Timesteps 1600, Score.Agents: 284.91, Avg.Score: 250.24, Time: 01:32:48, Interval: 00:21\n",
      "Ep. 273, Timesteps 1600, Score.Agents: 264.72, Avg.Score: 250.80, Time: 01:33:08, Interval: 00:20\n",
      "Ep. 274, Timesteps 1600, Score.Agents: 266.67, Avg.Score: 251.27, Time: 01:33:29, Interval: 00:21\n",
      "Ep. 275, Timesteps 1600, Score.Agents: 291.52, Avg.Score: 251.99, Time: 01:33:49, Interval: 00:20\n",
      "Ep. 276, Timesteps 1600, Score.Agents: 260.79, Avg.Score: 252.48, Time: 01:34:10, Interval: 00:21\n",
      "Ep. 277, Timesteps 1600, Score.Agents: 248.53, Avg.Score: 252.73, Time: 01:34:30, Interval: 00:20\n",
      "Ep. 278, Timesteps 1600, Score.Agents: 277.88, Avg.Score: 253.35, Time: 01:34:50, Interval: 00:20\n",
      "Ep. 279, Timesteps 1600, Score.Agents: 275.24, Avg.Score: 253.87, Time: 01:35:10, Interval: 00:20\n",
      "Ep. 280, Timesteps 1600, Score.Agents: 265.35, Avg.Score: 254.49, Time: 01:35:30, Interval: 00:20\n",
      "Ep. 281, Timesteps 1600, Score.Agents: 253.11, Avg.Score: 254.76, Time: 01:35:50, Interval: 00:20\n",
      "Ep. 282, Timesteps 1600, Score.Agents: 284.64, Avg.Score: 255.40, Time: 01:36:11, Interval: 00:21\n",
      "Ep. 283, Timesteps 1600, Score.Agents: 286.80, Avg.Score: 255.94, Time: 01:36:32, Interval: 00:21\n",
      "Ep. 284, Timesteps 1600, Score.Agents: 281.20, Avg.Score: 256.80, Time: 01:36:52, Interval: 00:20\n",
      "Ep. 285, Timesteps 1600, Score.Agents: 278.22, Avg.Score: 257.28, Time: 01:37:13, Interval: 00:21\n",
      "Ep. 286, Timesteps 1600, Score.Agents: 283.10, Avg.Score: 257.76, Time: 01:37:33, Interval: 00:20\n",
      "Ep. 287, Timesteps 1600, Score.Agents: 271.18, Avg.Score: 258.12, Time: 01:37:53, Interval: 00:20\n",
      "Ep. 288, Timesteps 1600, Score.Agents: 267.30, Avg.Score: 258.39, Time: 01:38:13, Interval: 00:20\n",
      "Ep. 289, Timesteps 1600, Score.Agents: 289.60, Avg.Score: 258.95, Time: 01:38:34, Interval: 00:21\n",
      "Ep. 290, Timesteps 1600, Score.Agents: 284.34, Avg.Score: 259.59, Time: 01:38:54, Interval: 00:20\n",
      "Ep. 291, Timesteps 1600, Score.Agents: 277.21, Avg.Score: 260.09, Time: 01:39:15, Interval: 00:21\n",
      "Ep. 292, Timesteps 1600, Score.Agents: 286.35, Avg.Score: 260.61, Time: 01:39:35, Interval: 00:20\n",
      "Ep. 293, Timesteps 1600, Score.Agents: 260.04, Avg.Score: 260.80, Time: 01:39:55, Interval: 00:20\n",
      "Ep. 294, Timesteps 1600, Score.Agents: 286.39, Avg.Score: 261.39, Time: 01:40:15, Interval: 00:20\n",
      "Ep. 295, Timesteps 1600, Score.Agents: 279.32, Avg.Score: 261.90, Time: 01:40:35, Interval: 00:20\n",
      "Ep. 296, Timesteps 1600, Score.Agents: 280.04, Avg.Score: 262.28, Time: 01:40:55, Interval: 00:20\n",
      "Ep. 297, Timesteps 1600, Score.Agents: 290.11, Avg.Score: 262.81, Time: 01:41:15, Interval: 00:20\n",
      "Ep. 298, Timesteps 1600, Score.Agents: 287.63, Avg.Score: 263.27, Time: 01:41:36, Interval: 00:21\n",
      "Ep. 299, Timesteps 1600, Score.Agents: 293.74, Avg.Score: 263.83, Time: 01:41:56, Interval: 00:20\n",
      "Saving model, i_episode:  300 \n",
      "\n",
      "Ep. 300, Timesteps 1600, Score.Agents: 301.38, Avg.Score: 264.36, Time: 01:42:17, Interval: 00:21\n",
      "Ep. 301, Timesteps 1600, Score.Agents: 301.68, Avg.Score: 264.88, Time: 01:42:37, Interval: 00:20\n",
      "Ep. 302, Timesteps 1600, Score.Agents: 299.87, Avg.Score: 265.39, Time: 01:42:57, Interval: 00:20\n",
      "Ep. 303, Timesteps 1600, Score.Agents: 303.40, Avg.Score: 266.05, Time: 01:43:17, Interval: 00:20\n",
      "Ep. 304, Timesteps 1600, Score.Agents: 269.61, Avg.Score: 266.39, Time: 01:43:38, Interval: 00:21\n",
      "Ep. 305, Timesteps 1600, Score.Agents: 301.68, Avg.Score: 267.14, Time: 01:43:58, Interval: 00:20\n",
      "Ep. 306, Timesteps 1600, Score.Agents: 303.51, Avg.Score: 267.68, Time: 01:44:18, Interval: 00:20\n",
      "Ep. 307, Timesteps 1600, Score.Agents: 284.72, Avg.Score: 268.08, Time: 01:44:39, Interval: 00:21\n",
      "Ep. 308, Timesteps 1600, Score.Agents: 290.40, Avg.Score: 268.42, Time: 01:44:59, Interval: 00:20\n",
      "Ep. 309, Timesteps 1600, Score.Agents: 300.49, Avg.Score: 268.83, Time: 01:45:20, Interval: 00:21\n",
      "Ep. 310, Timesteps 1600, Score.Agents: 306.45, Avg.Score: 269.54, Time: 01:45:40, Interval: 00:20\n",
      "Ep. 311, Timesteps 1600, Score.Agents: 298.53, Avg.Score: 270.18, Time: 01:46:00, Interval: 00:20\n",
      "Ep. 312, Timesteps 1600, Score.Agents: 306.23, Avg.Score: 270.77, Time: 01:46:20, Interval: 00:20\n",
      "Ep. 313, Timesteps 1600, Score.Agents: 301.24, Avg.Score: 271.34, Time: 01:46:41, Interval: 00:21\n",
      "Ep. 314, Timesteps 1600, Score.Agents: 297.42, Avg.Score: 271.88, Time: 01:47:01, Interval: 00:20\n",
      "Ep. 315, Timesteps 1600, Score.Agents: 304.58, Avg.Score: 272.50, Time: 01:47:21, Interval: 00:20\n",
      "Ep. 316, Timesteps 1600, Score.Agents: 303.67, Avg.Score: 272.97, Time: 01:47:42, Interval: 00:21\n",
      "Ep. 317, Timesteps 1600, Score.Agents: 300.57, Avg.Score: 273.34, Time: 01:48:02, Interval: 00:20\n",
      "Ep. 318, Timesteps 1600, Score.Agents: 300.75, Avg.Score: 273.84, Time: 01:48:22, Interval: 00:20\n",
      "Ep. 319, Timesteps 1600, Score.Agents: 273.55, Avg.Score: 274.06, Time: 01:48:42, Interval: 00:20\n",
      "Ep. 320, Timesteps 1600, Score.Agents: 301.76, Avg.Score: 274.54, Time: 01:49:02, Interval: 00:20\n",
      "Ep. 321, Timesteps 1600, Score.Agents: 301.44, Avg.Score: 274.95, Time: 01:49:23, Interval: 00:21\n",
      "Ep. 322, Timesteps 1600, Score.Agents: 304.61, Avg.Score: 275.40, Time: 01:49:43, Interval: 00:20\n",
      "Ep. 323, Timesteps 1600, Score.Agents: 303.47, Avg.Score: 275.81, Time: 01:50:03, Interval: 00:20\n",
      "Ep. 324, Timesteps 1600, Score.Agents: 298.52, Avg.Score: 276.32, Time: 01:50:24, Interval: 00:21\n",
      "Ep. 325, Timesteps 1600, Score.Agents: 306.10, Avg.Score: 276.80, Time: 01:50:44, Interval: 00:20\n",
      "Ep. 326, Timesteps 1600, Score.Agents: 303.74, Avg.Score: 277.19, Time: 01:51:05, Interval: 00:21\n",
      "Ep. 327, Timesteps 1600, Score.Agents: 287.06, Avg.Score: 277.46, Time: 01:51:25, Interval: 00:20\n",
      "Ep. 328, Timesteps 1600, Score.Agents: 291.27, Avg.Score: 277.78, Time: 01:51:46, Interval: 00:21\n",
      "Ep. 329, Timesteps 1600, Score.Agents: 304.16, Avg.Score: 278.41, Time: 01:52:06, Interval: 00:20\n",
      "Saving model, i_episode:  330 \n",
      "\n",
      "Ep. 330, Timesteps 1600, Score.Agents: 284.97, Avg.Score: 278.78, Time: 01:52:26, Interval: 00:20\n",
      "Ep. 331, Timesteps 1600, Score.Agents: 298.54, Avg.Score: 279.30, Time: 01:52:46, Interval: 00:20\n",
      "Ep. 332, Timesteps 1600, Score.Agents: 303.44, Avg.Score: 279.78, Time: 01:53:06, Interval: 00:20\n",
      "Ep. 333, Timesteps 1600, Score.Agents: 297.63, Avg.Score: 280.14, Time: 01:53:27, Interval: 00:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 334, Timesteps 1600, Score.Agents: 294.96, Avg.Score: 280.45, Time: 01:53:48, Interval: 00:21\n",
      "Ep. 335, Timesteps 1600, Score.Agents: 297.90, Avg.Score: 280.74, Time: 01:54:08, Interval: 00:20\n",
      "Ep. 336, Timesteps 1600, Score.Agents: 300.45, Avg.Score: 281.04, Time: 01:54:29, Interval: 00:21\n",
      "Ep. 337, Timesteps 1600, Score.Agents: 302.93, Avg.Score: 281.35, Time: 01:54:50, Interval: 00:21\n",
      "Ep. 338, Timesteps 1600, Score.Agents: 292.29, Avg.Score: 281.77, Time: 01:55:10, Interval: 00:20\n",
      "Ep. 339, Timesteps 1600, Score.Agents: 302.98, Avg.Score: 282.26, Time: 01:55:30, Interval: 00:20\n",
      "Ep. 340, Timesteps 1600, Score.Agents: 288.82, Avg.Score: 282.45, Time: 01:55:51, Interval: 00:21\n",
      "Ep. 341, Timesteps 1600, Score.Agents: 286.83, Avg.Score: 282.64, Time: 01:56:11, Interval: 00:20\n",
      "Ep. 342, Timesteps 1600, Score.Agents: 308.04, Avg.Score: 283.03, Time: 01:56:31, Interval: 00:20\n",
      "Ep. 343, Timesteps 1600, Score.Agents: 283.99, Avg.Score: 283.38, Time: 01:56:52, Interval: 00:21\n",
      "Ep. 344, Timesteps 1600, Score.Agents: 297.39, Avg.Score: 283.72, Time: 01:57:12, Interval: 00:20\n",
      "Ep. 345, Timesteps 1600, Score.Agents: 300.11, Avg.Score: 284.09, Time: 01:57:32, Interval: 00:20\n",
      "Ep. 346, Timesteps 1600, Score.Agents: 283.88, Avg.Score: 284.43, Time: 01:57:53, Interval: 00:21\n",
      "Ep. 347, Timesteps 1600, Score.Agents: 294.97, Avg.Score: 284.77, Time: 01:58:13, Interval: 00:20\n",
      "Ep. 348, Timesteps 1600, Score.Agents: 291.46, Avg.Score: 285.07, Time: 01:58:34, Interval: 00:21\n",
      "Ep. 349, Timesteps 1600, Score.Agents: 307.44, Avg.Score: 285.62, Time: 01:58:54, Interval: 00:20\n",
      "Ep. 350, Timesteps 1600, Score.Agents: 276.82, Avg.Score: 285.75, Time: 01:59:14, Interval: 00:20\n",
      "Ep. 351, Timesteps 1600, Score.Agents: 290.21, Avg.Score: 286.02, Time: 01:59:35, Interval: 00:21\n",
      "Ep. 352, Timesteps 1600, Score.Agents: 299.09, Avg.Score: 286.45, Time: 01:59:55, Interval: 00:20\n",
      "Ep. 353, Timesteps 1600, Score.Agents: 296.58, Avg.Score: 286.79, Time: 02:00:15, Interval: 00:20\n",
      "Ep. 354, Timesteps 1600, Score.Agents: 285.27, Avg.Score: 286.95, Time: 02:00:36, Interval: 00:21\n",
      "Ep. 355, Timesteps 1600, Score.Agents: 302.77, Avg.Score: 287.19, Time: 02:00:56, Interval: 00:20\n",
      "Ep. 356, Timesteps 1600, Score.Agents: 286.64, Avg.Score: 287.36, Time: 02:01:16, Interval: 00:20\n",
      "Ep. 357, Timesteps 1600, Score.Agents: 280.71, Avg.Score: 287.40, Time: 02:01:36, Interval: 00:20\n",
      "Ep. 358, Timesteps 1600, Score.Agents: 282.70, Avg.Score: 287.48, Time: 02:01:56, Interval: 00:20\n",
      "Ep. 359, Timesteps 1600, Score.Agents: 292.02, Avg.Score: 287.79, Time: 02:02:17, Interval: 00:21\n",
      "Saving model, i_episode:  360 \n",
      "\n",
      "Ep. 360, Timesteps 1600, Score.Agents: 310.82, Avg.Score: 288.30, Time: 02:02:37, Interval: 00:20\n",
      "Ep. 361, Timesteps 1600, Score.Agents: 284.03, Avg.Score: 288.43, Time: 02:02:57, Interval: 00:20\n",
      "Ep. 362, Timesteps 1600, Score.Agents: 305.10, Avg.Score: 288.70, Time: 02:03:17, Interval: 00:20\n",
      "Ep. 363, Timesteps 1600, Score.Agents: 268.41, Avg.Score: 288.57, Time: 02:03:37, Interval: 00:20\n",
      "Ep. 364, Timesteps 1600, Score.Agents: 282.97, Avg.Score: 288.78, Time: 02:03:58, Interval: 00:21\n",
      "Ep. 365, Timesteps 1600, Score.Agents: 299.87, Avg.Score: 288.90, Time: 02:04:18, Interval: 00:20\n",
      "Ep. 366, Timesteps 1600, Score.Agents: 287.23, Avg.Score: 288.98, Time: 02:04:38, Interval: 00:20\n",
      "Ep. 367, Timesteps 1600, Score.Agents: 293.67, Avg.Score: 289.14, Time: 02:04:58, Interval: 00:20\n",
      "Ep. 368, Timesteps 1600, Score.Agents: 297.49, Avg.Score: 289.27, Time: 02:05:18, Interval: 00:20\n",
      "Ep. 369, Timesteps 1600, Score.Agents: 280.66, Avg.Score: 289.55, Time: 02:05:39, Interval: 00:21\n",
      "Ep. 370, Timesteps 1600, Score.Agents: 290.48, Avg.Score: 289.87, Time: 02:05:59, Interval: 00:20\n",
      "Ep. 371, Timesteps 1600, Score.Agents: 298.30, Avg.Score: 290.19, Time: 02:06:19, Interval: 00:20\n",
      "Ep. 372, Timesteps 1600, Score.Agents: 276.58, Avg.Score: 290.10, Time: 02:06:40, Interval: 00:21\n",
      "Ep. 373, Timesteps 1600, Score.Agents: 293.83, Avg.Score: 290.39, Time: 02:07:00, Interval: 00:20\n",
      "Ep. 374, Timesteps 1600, Score.Agents: 290.18, Avg.Score: 290.63, Time: 02:07:21, Interval: 00:21\n",
      "Ep. 375, Timesteps 1600, Score.Agents: 294.66, Avg.Score: 290.66, Time: 02:07:41, Interval: 00:20\n",
      "Ep. 376, Timesteps 1600, Score.Agents: 266.52, Avg.Score: 290.72, Time: 02:08:02, Interval: 00:21\n",
      "Ep. 377, Timesteps 1600, Score.Agents: 269.28, Avg.Score: 290.93, Time: 02:08:22, Interval: 00:20\n",
      "Ep. 378, Timesteps 1600, Score.Agents: 290.65, Avg.Score: 291.05, Time: 02:08:42, Interval: 00:20\n",
      "Ep. 379, Timesteps 1600, Score.Agents: 297.91, Avg.Score: 291.28, Time: 02:09:02, Interval: 00:20\n",
      "Ep. 380, Timesteps 1600, Score.Agents: 297.58, Avg.Score: 291.60, Time: 02:09:22, Interval: 00:20\n",
      "Ep. 381, Timesteps 1600, Score.Agents: 296.62, Avg.Score: 292.04, Time: 02:09:43, Interval: 00:21\n",
      "Ep. 382, Timesteps 1600, Score.Agents: 287.43, Avg.Score: 292.07, Time: 02:10:03, Interval: 00:20\n",
      "Ep. 383, Timesteps 1600, Score.Agents: 304.44, Avg.Score: 292.24, Time: 02:10:23, Interval: 00:20\n",
      "Ep. 384, Timesteps 1600, Score.Agents: 299.86, Avg.Score: 292.43, Time: 02:10:44, Interval: 00:21\n",
      "Ep. 385, Timesteps 1600, Score.Agents: 294.55, Avg.Score: 292.59, Time: 02:11:05, Interval: 00:21\n",
      "Ep. 386, Timesteps 1600, Score.Agents: 304.36, Avg.Score: 292.80, Time: 02:11:25, Interval: 00:20\n",
      "Ep. 387, Timesteps 1600, Score.Agents: 307.25, Avg.Score: 293.17, Time: 02:11:45, Interval: 00:20\n",
      "Ep. 388, Timesteps 1600, Score.Agents: 299.69, Avg.Score: 293.49, Time: 02:12:06, Interval: 00:21\n",
      "Ep. 389, Timesteps 1600, Score.Agents: 306.87, Avg.Score: 293.66, Time: 02:12:26, Interval: 00:20\n",
      "Saving model, i_episode:  390 \n",
      "\n",
      "Ep. 390, Timesteps 1600, Score.Agents: 304.80, Avg.Score: 293.87, Time: 02:12:47, Interval: 00:21\n",
      "Ep. 391, Timesteps 1600, Score.Agents: 313.38, Avg.Score: 294.23, Time: 02:13:07, Interval: 00:20\n",
      "Ep. 392, Timesteps 1600, Score.Agents: 308.09, Avg.Score: 294.45, Time: 02:13:27, Interval: 00:20\n",
      "Ep. 393, Timesteps 1600, Score.Agents: 302.76, Avg.Score: 294.87, Time: 02:13:49, Interval: 00:22\n",
      "Ep. 394, Timesteps 1600, Score.Agents: 311.45, Avg.Score: 295.12, Time: 02:14:12, Interval: 00:23\n",
      "Ep. 395, Timesteps 1600, Score.Agents: 309.66, Avg.Score: 295.43, Time: 02:14:34, Interval: 00:22\n",
      "Ep. 396, Timesteps 1600, Score.Agents: 296.54, Avg.Score: 295.59, Time: 02:14:55, Interval: 00:21\n",
      "Ep. 397, Timesteps 1600, Score.Agents: 299.02, Avg.Score: 295.68, Time: 02:15:16, Interval: 00:21\n",
      "Ep. 398, Timesteps 1600, Score.Agents: 301.43, Avg.Score: 295.82, Time: 02:15:37, Interval: 00:21\n",
      "Ep. 399, Timesteps 1600, Score.Agents: 313.09, Avg.Score: 296.01, Time: 02:15:57, Interval: 00:20\n",
      "Ep. 400, Timesteps 1600, Score.Agents: 314.30, Avg.Score: 296.14, Time: 02:16:20, Interval: 00:23\n",
      "Ep. 401, Timesteps 1600, Score.Agents: 314.45, Avg.Score: 296.27, Time: 02:16:40, Interval: 00:20\n",
      "Ep. 402, Timesteps 1600, Score.Agents: 293.69, Avg.Score: 296.21, Time: 02:17:00, Interval: 00:20\n",
      "Ep. 403, Timesteps 1600, Score.Agents: 312.40, Avg.Score: 296.30, Time: 02:17:21, Interval: 00:21\n",
      "Ep. 404, Timesteps 1600, Score.Agents: 312.32, Avg.Score: 296.72, Time: 02:17:41, Interval: 00:20\n",
      "Ep. 405, Timesteps 1600, Score.Agents: 308.56, Avg.Score: 296.79, Time: 02:18:01, Interval: 00:20\n",
      "Ep. 406, Timesteps 1600, Score.Agents: 308.10, Avg.Score: 296.84, Time: 02:18:21, Interval: 00:20\n",
      "Ep. 407, Timesteps 1600, Score.Agents: 298.03, Avg.Score: 296.97, Time: 02:18:42, Interval: 00:21\n",
      "Ep. 408, Timesteps 1600, Score.Agents: 305.40, Avg.Score: 297.12, Time: 02:19:02, Interval: 00:20\n",
      "Ep. 409, Timesteps 1600, Score.Agents: 302.53, Avg.Score: 297.14, Time: 02:19:22, Interval: 00:20\n",
      "Ep. 410, Timesteps 1600, Score.Agents: 309.42, Avg.Score: 297.17, Time: 02:19:43, Interval: 00:21\n",
      "Ep. 411, Timesteps 1600, Score.Agents: 316.48, Avg.Score: 297.35, Time: 02:20:03, Interval: 00:20\n",
      "Ep. 412, Timesteps 1600, Score.Agents: 302.61, Avg.Score: 297.32, Time: 02:20:24, Interval: 00:21\n",
      "Ep. 413, Timesteps 1600, Score.Agents: 295.93, Avg.Score: 297.26, Time: 02:20:44, Interval: 00:20\n",
      "Ep. 414, Timesteps 1600, Score.Agents: 317.01, Avg.Score: 297.46, Time: 02:21:04, Interval: 00:20\n",
      "Ep. 415, Timesteps 1600, Score.Agents: 305.79, Avg.Score: 297.47, Time: 02:21:24, Interval: 00:20\n",
      "Ep. 416, Timesteps 1600, Score.Agents: 307.45, Avg.Score: 297.51, Time: 02:21:44, Interval: 00:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep. 417, Timesteps 1600, Score.Agents: 300.34, Avg.Score: 297.51, Time: 02:22:04, Interval: 00:20\n",
      "Ep. 418, Timesteps 1600, Score.Agents: 285.32, Avg.Score: 297.35, Time: 02:22:25, Interval: 00:21\n",
      "Ep. 419, Timesteps 1600, Score.Agents: 302.57, Avg.Score: 297.64, Time: 02:22:45, Interval: 00:20\n",
      "Saving model, i_episode:  420 \n",
      "\n",
      "Ep. 420, Timesteps 1600, Score.Agents: 305.35, Avg.Score: 297.68, Time: 02:23:05, Interval: 00:20\n",
      "Ep. 421, Timesteps 1600, Score.Agents: 306.09, Avg.Score: 297.72, Time: 02:23:26, Interval: 00:21\n",
      "Ep. 422, Timesteps 1600, Score.Agents: 300.48, Avg.Score: 297.68, Time: 02:23:46, Interval: 00:20\n",
      "Ep. 423, Timesteps 1600, Score.Agents: 304.90, Avg.Score: 297.70, Time: 02:24:07, Interval: 00:21\n",
      "Ep. 424, Timesteps 1600, Score.Agents: 278.19, Avg.Score: 297.49, Time: 02:24:28, Interval: 00:21\n",
      "Ep. 425, Timesteps 1600, Score.Agents: 301.05, Avg.Score: 297.44, Time: 02:24:48, Interval: 00:20\n",
      "Ep. 426, Timesteps 1600, Score.Agents: 299.23, Avg.Score: 297.40, Time: 02:25:08, Interval: 00:20\n",
      "Ep. 427, Timesteps 1600, Score.Agents: 303.30, Avg.Score: 297.56, Time: 02:25:28, Interval: 00:20\n",
      "Ep. 428, Timesteps 1600, Score.Agents: 316.41, Avg.Score: 297.81, Time: 02:25:48, Interval: 00:20\n",
      "Ep. 429, Timesteps 1600, Score.Agents: 295.04, Avg.Score: 297.72, Time: 02:26:08, Interval: 00:20\n",
      "Ep. 430, Timesteps 1600, Score.Agents: 293.99, Avg.Score: 297.81, Time: 02:26:28, Interval: 00:20\n",
      "Ep. 431, Timesteps 1600, Score.Agents: 307.55, Avg.Score: 297.90, Time: 02:26:49, Interval: 00:21\n",
      "Ep. 432, Timesteps 1600, Score.Agents: 301.02, Avg.Score: 297.88, Time: 02:27:10, Interval: 00:21\n",
      "Ep. 433, Timesteps 1600, Score.Agents: 300.46, Avg.Score: 297.91, Time: 02:27:30, Interval: 00:20\n",
      "Ep. 434, Timesteps 1600, Score.Agents: 307.84, Avg.Score: 298.03, Time: 02:27:50, Interval: 00:20\n",
      "Ep. 435, Timesteps 1600, Score.Agents: 324.99, Avg.Score: 298.30, Time: 02:28:10, Interval: 00:20\n",
      "Ep. 436, Timesteps 1600, Score.Agents: 316.43, Avg.Score: 298.46, Time: 02:28:31, Interval: 00:21\n",
      "Ep. 437, Timesteps 1600, Score.Agents: 312.66, Avg.Score: 298.56, Time: 02:28:51, Interval: 00:20\n",
      "Ep. 438, Timesteps 1600, Score.Agents: 325.93, Avg.Score: 298.90, Time: 02:29:11, Interval: 00:20\n",
      "Ep. 439, Timesteps 1600, Score.Agents: 293.70, Avg.Score: 298.81, Time: 02:29:32, Interval: 00:21\n",
      "Ep. 440, Timesteps 1600, Score.Agents: 298.70, Avg.Score: 298.90, Time: 02:29:52, Interval: 00:20\n",
      "Ep. 441, Timesteps 1600, Score.Agents: 308.48, Avg.Score: 299.12, Time: 02:30:12, Interval: 00:20\n",
      "Ep. 442, Timesteps 1600, Score.Agents: 312.69, Avg.Score: 299.17, Time: 02:30:32, Interval: 00:20\n",
      "Ep. 443, Timesteps 1600, Score.Agents: 321.17, Avg.Score: 299.54, Time: 02:30:52, Interval: 00:20\n",
      "Ep. 444, Timesteps 1600, Score.Agents: 305.03, Avg.Score: 299.62, Time: 02:31:13, Interval: 00:21\n",
      "Ep. 445, Timesteps 1600, Score.Agents: 302.35, Avg.Score: 299.64, Time: 02:31:33, Interval: 00:20\n",
      "Ep. 446, Timesteps 1600, Score.Agents: 299.40, Avg.Score: 299.79, Time: 02:31:54, Interval: 00:21\n",
      "Ep. 447, Timesteps 1600, Score.Agents: 308.65, Avg.Score: 299.93, Time: 02:32:14, Interval: 00:20\n",
      "Ep. 448, Timesteps 1600, Score.Agents: 305.12, Avg.Score: 300.07, Time: 02:32:34, Interval: 00:20\n",
      "Ep. 449, Timesteps 1600, Score.Agents: 306.68, Avg.Score: 300.06, Time: 02:32:54, Interval: 00:20\n",
      "Saving model, i_episode:  450 \n",
      "\n",
      "Ep. 450, Timesteps 1600, Score.Agents: 322.10, Avg.Score: 300.51, Time: 02:33:14, Interval: 00:20\n",
      "Environment solved with Average Score:  300.5118793052797\n"
     ]
    }
   ],
   "source": [
    "scores, avg_scores = ppo_vec_env_train(envs, agent, policy, num_processes, max_steps, rollouts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(model=policy,directory='dir_save',filename='we0',suffix='final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  451 , len of avg_scores:  451\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAEGCAYAAACdE2QGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXiU1d3G8e+Z7CE7CQQSQlgSIOy7qCgqKiqCICjuWlq01rpUa6tWq21tfV1qbasoWje0ioJbFUVREUVQgmxh38IaIBDInklm5rx/zCQECEsgyQxwf65rrsyc55mZX0Zkbs45zznGWouIiIjIic7h7wJEREREGoJCjYiIiJwUFGpERETkpKBQIyIiIicFhRoRERE5KQT7uwB/SUxMtOnp6f4uQ0TkhLJgwYJd1tokf9chUpdTNtSkp6eTnZ3t7zJERE4oxpiN/q5B5FA0/CQiIiInBYUaEREROSko1IiIiMhJ4ZSdUyMiIoFhwYIFLYKDg18CuqF/bMuheYAcl8v18759++6s6wSFGhER8avg4OCXkpOTuyQlJe1xOBzakFDq5PF4TH5+ftb27dtfAkbUdY4SsYiI+Fu3pKSkIgUaORyHw2GTkpIK8fbo1X1OE9YjIiJSF4cCjRwN35+TQ2YXhRoRkRNcRZWbr1ftpMTp4u0fN+Hx1J0PrLXsKKpo4upEmo5CjYhIEygsq2JvWWWdxwpKK1mzo5i8wnLmrd99xNean1tARZW75vEL36znplfm0+2PM/j9e0tZsGkP5ZVurnh+LlPmb6o5b9m2Ik7725d8vmz78f9CJ6Hf/e53yR07duyamZmZ1blz56yvvvqqmb9rkvrRRGERaXKlThdhwQ6q3JaI0KBDnud0uTEYQoOP/O8vay0/bihgQLsEjDEAlFe6yS92ktY88qDzXW4Pj89YxdUD0khPPL7vrm17ywkPCWL5tiLuf38pV/Zvw4q8Irq0iuHWIR145H/LefX7XBwG2jZvRnhIEOP6t+Hyvqm4PZYxE79nR1EFLWPCWb+rlH9d1ZtLurfC4TAUllcREx5c8zst2LiHsc/P5eaz2nPVgDRax0Xw1cod+9WzbGshr8zZwI+5BSzavJfRfVKZuXwHny/fQYjDwcB2zY/r9z0ZzZw5s9mMGTPili5dujwiIsLm5eUFO51Oc6yvV1VVRUhISEOWKEdBoUZEGsWCjQX8b3EeJU4XsREh7C2rIi4yhCv6teHKSXPZW1ZFVFgwl/ZszXdr8xnYrjlnZyaxekcxc9buIr/EyeaCcmIjQph4bR/+/dVaxvZL5YvlOxjRszV92ybw6vcbKCitpLzSTedWMTz26Uoev7wHCc1CObdzC2585Ud+2FBAziMXsnVPOZUuDx8s2soDF3dh7vrdTJq9nkmz1zOyV2vuuaATVW4P7ZOi8HgsVR4PlS4P0eH7fzEVllcRHuJgxrIdvDFvI4M7JvL3mauxtUZ8npixCoCPl+RR5fbw2txc2iREsLmgnA27SgH440fLeOSjpYTgIggPDjzs2lVIPB4efmsWa9alsiG/mAUbduEwHromR3FR1yQ+z9lGhilmUfY2vv22klCHBY+bv/VPITQIpvyQy2efLCcEF1fGGIpLSnnumQVs272XKKr4W6sIYp1ZENm2qf4onBC2bt0akpCQ4IqIiLAArVq1cgF88803kXfeeWdaWVmZIzQ01M6ePXtVWFiYvf7669suWbIkMigoiMcff3zzpZdeWvzPf/6z+aeffhrrdDodZWVljnnz5q1+8MEHW77//vsJlZWV5pJLLtn79NNPb/Pvb3pyM9aemnOz+vXrZ7X3k0jdCkoryS920ik5+rDnOV1uwoL372kpq3SxZkcJ17z0AyVO10HPaRUbTl6hd15H82ah7C6tJDEqjF0lzv3OS4mLYOve8mP+HWq/ZqeW0azaUVxzbOotg/gsZzsvfbcBsITiItRhsR43mc1DSG4GRUXFuJylnN8xhtKyEoZ1iqWopIQp368m1FYSjvcWZrw/I3HSJsrSvUUIhYWFhFNBYWEhoZ5yIh1VtGzmoLCkjGDcRAV7sO4qHHiO+fc7Lle/C5kXHNNTjTELrLX9GrKcxYsX5/bs2XMXwG+nLm6zenvxwV1rxyEzObrsiTE9Nx/unMLCQsfAgQM7V1RUOM4888yiq666quC8884r7dixY7c333xz3dlnn11WUFDgiI6O9vzlL39puWzZsoipU6fmLly4MPziiy/OWLduXc5LL72U8Oijj6YsWbJkWcuWLd3vvfdezLvvvhv/5ptvbrTWMnTo0I733nvv9osuuqikIX+/U83ixYsTe/bsmV7XMfXUiJyCXG4POduKeOjDHJ4c25OcrYVc1iuF5XlFbCoo4/8+W8nG3WWs/+vFOBzeHvgvV+zgtbkb+fW5HXEYaB0XwblPfsM5nZOIDguhWVgwLWLCmLl8B9kb9xAfGcK3957Dpzl5/HX6ypr3zius4O9X9KRbSixpCZF8uWIn53Vpwdx1u7np1fkA/HD/ecRGhLCpoIyXv9vA2/P3fR/dOqQDxkB5hZOxXaN5+J3vqSjeTawpJZZSYkyZ92dFKTHBpcSYcsILnESEVBJuKomgkrjJLn7hLuf2sAqiKMdhav3jrtR3q7bG93Or90f/A/7W9GAgOBwTEokJbQblkTRvFgmhUURExVHiCaVFUgKO4FC27SinuMpwWkYy1oSwrdRNcnw0jqBgyqpgXu4ezshoQVGFh398tQ43Dv42ugf5pS7+9tkaPBjuGZaFcTiYkr2NjsmxXNKjNUFBwRhHMBjDzFW7WLG9lJ+dnUmzyGYQFEZBpSEqshmhYREQHAohmipyoNjYWE9OTs7yzz77LPrLL7+MvuGGGzrcddddeS1atKg6++yzywASEhI8AN9//33Ur3/9650AvXv3rmjdunXl0qVLwwEGDx5c1LJlSzfAZ599FjN79uyYrKysLICysjLHypUrwxVqGo9CjchJrLCsivveX0Kb+Ejim4Vy4+npLNtWyPjXstlbVgXABU/PBiAtIZIxz8/d7/nvLdzKwk17uOXsDvxu2hJ2lVQye3U+AB2SmlFe5Wb60oMnnbZJiOD5a/vSJiGS3mnxNe1nZSbRJTmaUb1TauaIXNI9GSoK6RddQB+zmuamiJZrd0PpLjLLdvMH105ubLmdpJAyivbsIm1ZFUHOQqgsgYUwBSDs4N+9glBCIuPILQmijDAyUlqwYlcYpUHhrCn1Hu+Y0oLvtlThtCF4jIPeac1p0yKBV+dvp8KGUoH3lhQfx5oCFxWEcu/wXgzMbM2L87bzs7O7EBcdBabuqRfxvlu1rrXuO4DWtR5HAuf67icBlQWLyWodg+nbjiRreX/6dACeOesSAO4+s863ZGhHGHpAW0LdpwakI/WoNKbg4GCGDx9ePHz48OIePXqUP//880nGmIOGMw43whEZGempfd6dd96Z99vf/nZXI5UsBwjIUGOMaQO8DiTjXRZ5krX2GWPMw8AvgHzfqfdba6f7nnMfMB5wA7dba2c0eeEi9WStrflyr+bxWIzhoPaPFm/jte9zWbatkNduGsArc3KJbxbK5X1S6Jde99fWnz9Zvl/o2F5Ywac5eTWBBqBjiyjW7izhx9yCg57/+2lLcHksXyzfwa6SSm4d0oGiiiremLeJdfne7owbT0/n1e9zAYgOC+bO8zO58fR0glxlULCBzlXbuMAxn0RTxF/bt4TSnfB2HhT7bqX54HERDbxXHU4+8v0MiSQqIoHOEXEQHkvz5l0hIg7CvY9r7tdqs+GxmIh4QoPCcDgML0xdTLvEKLoP6UBv3+e7dfE2+qbE0LFFNJF5RVz0zLcMzkjk9+MHAnBhpx10S4nljrcXMrZvG3YUVzDtM+88mV49exMdFcZvLk07iv/Cx+6JsT1r7htj+OOlWcRFauJpY1m8eHGYw+Gge/fuToCFCxdGZGRkVMyaNSv2m2++iTz77LPL9uzZ44iKivKceeaZJW+88UbCiBEjipcsWRKWl5cX2qNHj4offvhhv2Gziy66qOjhhx9uPWHChILY2FjPhg0bQkJDQ21KSsrB47LSIAIy1AAu4G5r7U/GmGhggTHmC9+xp621T9Y+2RiTBYzD+w+h1sBMY0ymtdaNiJ/lFZYzNXsL4we3IzLU+79cpcvD+Nfm0yEpit9f1JlnvlzDqu3FZLSMIjt3D7tLnFw/KJ2bzkinosrDbf/9iS9X7iQ8xEFFlYdb3/yJ3aXey4Pf+nETcZEhnNkxkYLSSrbuLSe/2InLY6l0eRiQnkB8sxB2l1TWhI+/XNaNP3yQQ+fkaD69YzA9HvmciV+vO6h2l2+9k53FTsAypE0wAxLd3Jrq5pXPs+mV4OKSiB8ZEJVDiLOAgUkeYrL3wDf5UFUGQDQwKdT3grOAiHiIbuW9tciCqBbQLBEim7PbRhMUlURcYjJEJkJo/adWVEfB6uulHh/Tc7/jDofhst4pNY/bJzWjT1oc489sV9N2XpeWALw9YRAACzftAeDi7skkRtXRLdQEbjqj3ZFPkmNWVFQUdPvtt6cVFRUFBQUF2fT0dOdrr722cfXq1btuv/32tIqKCkd4eLhn9uzZq++9996d1113XdvMzMysoKAgXnjhhdzqCca1jR49umjZsmXh/fv37wzeXpw333xzg0JN4zkhJgobYz4E/g2cAZTUEWruA7DW/s33eAbwsLV27oGvVU0ThaUx5GwtpNLtoVdqHA6HYV1+CZf+6zvKKt08MaYHY/u1AeC5WWt53Pcv/y6tYliRV1Tn671wXV8+WZLH/5Zs4zdDM7llSAfO/L+v2FHkpG/beHqmxvHynA2HrGdwRiKv3jSAIIdh8ea9/OL1bLq2juHlG/vz3Kx1XJDVkowWUfxs4ufs2LyG67IcDG7pJsyZz49LVhBWkU+3mHJsyQ6aU0SIqePfCY4QnGEJbHFGkp6WRlB0C2jWAqKSoFlSrfstvI+DQw9+jRNAYXkVsRHqKWnsicIiR3JCTxQ2xqQDvYEf8Iaa24wx1wPZeHtz9gApwLxaT9viazvwtSYAEwDS0hq361hOPtZaCsuriIv0fikXV3gvSV67swS3tXywcBvPf+Pt7bjx9HSGdUtm3KR9fyw/XLSNibPWkZ7YjK9W7qR9UjN2FTvZUlDGn0Z2pUdqHJc9O4cB6Qm8eEM/ej7yOTdPXgDAbed05NfnZQBwaY/WvPTdBm4/LwOPx/LynA1cNSCNt37cxMOXZvHxkjz+eGlXdpU4Gdg+gSCHAbeLnpG7+WGsB7M7G6a/w6/2boLlm6BwMy9Xlnjnpazz3YyDc0ITWGeiMNFtsS27Mbc0nLN6ZXl7WGJa+wJLIoTFEGYMHZryP4YfKNCIBL6A7qkxxkQB3wCPWmvfM8a0BHYBFvgz0Mpa+zNjzLPAXGvtG77n/QeYbq2ddqjXVk+NHI0te8oICw4iO7eAV+bksnDzHiaPH0hheRU3T17ANQPTePOHfSu2towJo3ebeD6rtWLr2L6p7Ch21kywTYwKpajCxX9u6McZHRIB75CItZb/LcljcMdE4puFcvEz37I8r4hrT0vjD5dkER7ivXTa5fbgtrbmUupl2wrJahVDhbOSiNItULABCtZBwXrYvc57f+8m8NTq8Q6Phbg0iGsLsW1899t470e3gmaJWONg0ea99GoTd9D8Hjl1qadG/O2E7KkxxoQA04A3rbXvAVhrd9Q6/iLwse/hFqBNraenAlrgSOqtospNSJCDEqeLL1fs4DfvLK45FuwwuDyWq1+cVxMoqgPNhV1bEhUWwvWD2pLevBmllS6+XbOLsGAHT4ztydQFW1iQW8C1g9py30Vd6pwgbIxhRM9918NMHj8Ap8tD67iI/c4Lti6Cd62C7Uth+1K67loDBeuIODC4hEZBQntI7gFdR3nvJ3SAxAyIbH7IK3Zq6oH9rlwSEQl0ARlqjPdv+/8AK6y1f6/V3spam+d7OArI8d3/CPivMebveCcKZwA/NmHJcgL6YvkO/v31WjbtLqV3WjzBDsOs1flUug5eEC0mPJiFD13Apzl53PbfhZRXufn31b157ut1XD+oLeMG7D+c+eL1/Rg3aR6jfBNSx/RNZUzf1JrjR9Pz0TwqDMr3Qu6CmgDD9iWwcyV4fFcvBUd4Q8qBwSWhvXcCrnpYROQUEpChBu/cmeuApcaYRb62+4GrjDG98A4/5QI3A1hrlxlj3gGW471y6le68knqsqvEyaTZ67njvAwe/CCH7b4di79aufOwz0tPbEaQwzCsazK3n5fBBVkt6ZYSy/Aeres8PzwkiA9+dUb9inM5vcFlSzZszYYt82FP7r7jUS2hZTcYdB4kd/cGmeYdwHHovZNERE4lARlqrLXfse/KzNqmH+Y5jwKPNlpRckLZVeLk7R83sXZnCd1T4xjZqzWvzsllZ3EF72RvIchh2FXiJCk6jLSESJZuKeTpK3uxcNMehnVLrlmE7qmxPbn73cWEBnkvEA4OcvCb8zOPv0BrvXNeti7YF2K2LwW3bxfn6NaQ2hf63ACtekDL7hDd8vjfV0TkJBaQoUbkaB2499B3a3bx9MzVxEaE1PS+fLBoG/M3FOw3eXfiLO9VSv8c15v+6fE4XR6ahQVzSY9WANx+XgZt4iNoEeNdk+Rodok+rKoKb8/Lprnen1uyody32F1IJLTuA6f9ElL6QWo/79VFItKkXn/99bgbbrihw08//bSsd+/eFf6uZ/v27UEjR47ssHTp0mZjxozZ/frrr9dclfDtt99Gjh8/Pr2iosJx7rnnFr788subHQ4HO3bsCBo1alT7rVu3hqWkpDg//PDD9UlJScc9cnH22Wd3nDZt2obExMRjfq2PP/44+qmnnmr59ddfrz3eeg5FoUZOWNOX5nHnlEVc0S+V8We2p11iM/708TJW7/Buq9I9JZZxA9rwwPs5NYFmUPvmTDirPR8u2krPNnGc1j4BYwzBQfuHluremIoqNxd1S+buCzrVr7iqctj8I2ycA7nfeUOM2wkYSOoMnS/eF2CSukCQ/lcU8be33347oU+fPiWTJ09O6N27t98vNomMjLR/+tOfti1evDgiJydnvysGbr311rbPPffcxnPPPbd0yJAhGVOnTo254ooriv74xz+2GjJkSPFf//rXNffff3/yQw89lDxx4sStx1vLN99802hBpCEd5z8/RZrGJ0vyuO+9Jfu1PfrJCipdHt6Yt4mR//6O+99fyuodJUSFBXPj6elMufk0ruq/bwLvVQPSeGvCaZzTuQX/GNebm85od8QJu+EhQUy8ti8dW0QdvkCX0xtevnoUXh4Gj6XB6yNg9hPelXUH/AKumgK/y4VfzYORz0K/m7xzYxRoRPyusLDQkZ2dHfXKK6/kvv/++zWX/V1yySXtp0yZElv9+PLLL09/9dVX44qLix0XX3xx+8zMzKxLLrmkfY8ePTrPnj37oCWwP/zww+guXbpkZWZmZo0dOza9vLzcAKSkpHS/6667WmdlZXXJzMzMWrhwYfiBz42JifFceOGFJeHh4ftdvbBx48aQkpISx9ChQ0sdDgfXXHPN7g8++CAe4LPPPou7+eabdwPcfPPNuz/99NODLmF0uVzcfPPNqd26deuSmZmZ9cQTTySCtyelX79+nc4///wOHTp06Hr11Venud3ejpmUlJTueXl5wUVFRY4hQ4Z07NSpU1ZGRkbXF198Mf5wv+fUqVNj2rVr17Vv376dpk6dGlddQ1FRkWPs2LHp3bp169KlS5esN954Iw4gOzs7vHv37l06d+6clZmZmbV06dJ6LeGtv00l4Flr+dV/fwIgOSaCS3oksyKvmK17y/nZGe2Ijwzh4yV5/PeHTZzTKYlnr+lTsx0BQFRYMCVOF1cNaHOot6g/jwd2LIX1s2D9N95hpaoyMA5o1QsG3gLpgyFtoHdNGBE5Oh/8qg07l9d/f4zDaZFVxmXPHnajzDfffDNuyJAhhT169HDGxcW5v/vuu8gzzzyz7MorryyYMmVK/JVXXllYUVFh5syZE/Paa69tfPzxx1vExcW5V69evXz+/PnhgwYN6nrga5aVlZmbb7653eeff76qR48ezlGjRqU/8cQTSQ899NBOgMTERNfy5ctXPPbYY0mPPfZYyylTpmw8ml9n48aNIa1atarZwK1t27aVeXl5IQC7d+8Obtu2bZWvvaqgoOCg7/l//OMfibGxse6cnJwV5eXlpn///p0vvfTSIoClS5c2W7hwYU5mZmblWWedlfH666/H33TTTXuqn/vee+/FJCcnV82aNWut7/2CDvV73nPPPfm33XZb+hdffLGqa9euzuHDh7evfp3777+/1TnnnFP07rvv5u7atSuoX79+XUaMGFH0r3/9K+nWW2/d8ctf/rKgoqLCuFz121FCoUYC3uw1+9bkenrmap6eubrm8ZBOSZyVmcSvz8ug0uWpc+7Laz8bQEFpJT1S4w46Vi+FW2DNF7D+a9jw7b45MYmdoPe10H4ItD3Du7miiJxQ3nnnnYQ77rhjJ8Dll19eMHny5IQzzzyzbMyYMYX33ntvWnl5uZk2bVrsgAEDiqOiouz3338fVX1+//79KzIzM8sOfM3FixeHp6amOnv06OEEuPHGG3c/++yzLYCdAFdfffUegAEDBpR99NFHR70oVF2L5tZngcyZM2fGrFy5MrL6PYuLi4OWL18eHhoaart3716alZVVCXDFFVcUfPvtt1G1Q02fPn3KH3jggTa//OUvU0aOHFk4bNiwkrlz50bU9XsOHTq0ODU11Vm9Seg111yz+6WXXkoCmDVrVsyMGTPi/vnPfyYDOJ1Os3bt2tBBgwaVPvnkk622bNkSOm7cuD3Vzz1aCjXiV1+v3Mnb8zcx4awO9EiNJSTIwda95cxcvoPU+Aie/Hw1a3YUkxIXgdPlISosiPFntuPBD5cB3n2Tqh1qMm/ftse4gJyr0tsDs/YLWDMT8ld426NbQ+YwaH82tDsbYlod2+uLyMGO0KPSGLZv3x40b968mNWrV0fcdtttuN1uY4yxEydO3BIZGWlPO+204vfeey9mypQp8VdddVUB1B0sDnSkc8LDwy1AcHCwdblcR51K0tPTq6p7ZgA2btwYmpycXAXQvHlz18aNG0Patm1btXHjxpCEhISDujqsteapp57adPnll++36dzHH38cXdeioLX16NHD+dNPPy2fNm1a7AMPPJAyc+bMotGjR+89VK2HClvWWqZOnbq2Z8+e+4WWPn36VAwePLj0/fffj73ooosyn3vuudwRI0YUH/LDOIDm1EiT215YwYuz17Nw0x5unryAGct2cPnE7xn7/FwKSisZ/s9v+eNHyxj/WjYr8ooYnJHIP8b14tt7z2Hmb87mukHpTPvl6dw6pANJ0Q28Y/LezZD9Mrx1NTzezjsvZt7z3g0Zz/8z3DoPfrMcRk2EnuMUaEROApMnT44fPXr07m3bti3dunXr0u3bty9JTU2t/Pzzz6MAxo0bV/Dqq68mzp8/P3r06NFFAKeffnrJ22+/HQ+wYMGC8NWrV0cc+Lq9evWq2Lp1a2hOTk4YwOuvv9588ODBR/0FfSht27atatasmefLL79s5vF4ePPNN5uPHDlyL8CFF16494UXXmgO8MILLzQfNmzYQYHj/PPPL5w4cWKS0+k0AEuWLAkrKipygHf4aeXKlaFut5upU6cmHFhvbm5uSHR0tOfWW28tuPPOO3csWrQo8lC/Z69evSq2bNkSumzZsjDwTsSufp1zzjmn6Kmnnmrp8XinC82ZMycCYPny5aFdunRx/uEPf9h5wQUX7F20aNFBn+vhqKdGmtwj/1vGpzneq5ESo8I4r3MLpmRvZtHmvQz9+zfsLa9i0nV9mbFsBxd1S2Zo1sHrs/RtG3/sPTC1WetdpXf5R7Dyk329MbFp0OMK6DgU2p0FYdHH/14iEpDefffd5vfee29e7baRI0fumTx5csKwYcNKRo0aVXTLLbe0Gzp06N7q3pXf/va3+VdccUV6ZmZmVrdu3co6depUHh8fv9/lzpGRkfb555/PHTt2bAe3203Pnj3L7rnnnvz61JaSktK9pKQkqKqqysyYMSNu+vTpq/v27Vvx3HPPbRw/fny7iooKc8455xSNHTu2EOCRRx7JGzVqVIe2bdsmtm7duvKDDz5Yd+Br3nXXXbtyc3PDunfv3sVaaxISEqqmT5++DqBXr14ld999d+rKlSsjBg4cWHzdddftF4oWLFgQcd9996U6HA6Cg4Ptc889t/FQv2dERIT917/+tXH48OEdExISXAMHDixZsWJFBMBjjz22bcKECWmdO3fOstaa1NRU59dff7128uTJCe+++27z4OBgm5SUVPW3v/2tXlehBfSGlo1JG1r6R+6uUi78x2ycLg/xkSE8ODyLYd2S+XLFTr5dk89nOdv5xeD2NTtSNwqPx7tWzIqPYMX/YO9G7wTftmd4h5UyzofETG0xIFIHbWjp5XK5qKysNJGRkXbZsmVhF1xwQea6detyqkPPiagp1pFpCCfkhpZy4nK63LwyJ5ekqDDCQ4KIiwyhosrN0zNXk7O1iLjIEL68+2xS4/dd4HBpz9Zc2rM1j4/p2ThFWQubf4ClU2Hlx1CcB44Q6HAOnHUPdLoEmjVvnPcWkZNOcXGxY/DgwZ2qqqqMtZann35644kcaE4WCjXSIKp7/Iwx/OXjFUyed+grEx8Z0XW/QNOoSvJhydvw0+uwa7V3A8iModBlBGReqMutReSYxMfHe3Jyclb4u46GNHz48OLhw4cf95wff1KokWOyansxt765gMEZSVwzMI2fv57NZb1SuKJ/G976cdNB5/dPj+eh4V3J2VbIiJ5NsAXAxrnww0RYOd27o3Wbgd4F77Iug7AjLKQnIk3N4/F4jMPhUE+HHJbH4zGA51DHFWrkqFW5PYx6bg5RYcHERYSyLr+UdfmlvPp9LgDPfLmGZ75cA8Bjo7vz+/eWEhMezB+GZ3FFP+/Cd91TG7FnxOOBNZ/Dd0/D5nkQkQADb4be10GLzo33viJyvHLy8/OzkpKSChVs5FA8Ho/Jz8+PBXIOdY5CjRyVz3K2A5acrfuWNbioW7K/VLIAACAASURBVDJLthTSoUUUN52Rzk2vzK85Nm5AGl1axdA9JRaHo5En3LqrIOc9mPMP2Lnce+XSxU9Cr2sgtImGuUTkmLlcrp9v3779pe3bt3dDS43IoXmAHJfL9fNDnaBQIzU8Hsukb9czuk8KLaK925DkbC3knncXs3K7d5g1yGH43bBOrNlRwr3DOhMXGUKIbzPIDX+7mLumLOK09t4Jtz3bNPLKupVlsHAyfP8vKNwMLbJg9IvQdRQEhRz5+SISEPr27bsTGOHvOuTEp1BzisvdVcrHS7ZxcfdW7Cmr5LFPV/LNqnzemnAaAHdOWcTanSU15w9sl8CEszrU+VrGGP4xrnfjF11WAPNfgh+eh7LdkDYILnkKMi7QZdgiIqcwhZpT1KLNe/l65U7mrtvNj7kFPPn5vv2U5q7fzdUvzuPWIR33CzQAt5xdd6BpEoVbYd5zkP0KVJV615Q5405oO8h/NYmISMBQqDkFeTyW6176gWKnd0uQ4T1a8fGSfYtpdkuJ4ft1u/l+3W5S4iJ45ab+BDkMlS7PfnstNZn81fD9M7B4ClgPdB8DZ9wBLQ/aFFdERE5hCjWnoB82FFDsdJESF8ENp7fl52e255ERXfm/z1aSlhDJbedm8Ou3FvK/xdv408iuZLb00xYBWxbAnKdhxccQHAb9boJBt0F8W//UIyIiAU3bJJxiCkoruXlyNgs37WXRHy8gKqzuXOt0uVm+rYjeaQ2wv1J9WAvrv/Zelr1htndxvAETYMDN3k0lRcSvGmObBJGGop6aU0Sp00VRRRUPvJ/D/Nw9ZLWKOWSgAQgLDmraQOPxePdi+u7vkLcYolvBBY9C3xu0maSIiByVkybUGGOGAc8AQcBL1trH/FxSQLn6xXks3lIIQFiwgz9fFiDzUayF1TPgq7/AjqXQvCOM+Ld3h+zgMH9XJyIiJ5CTItQYY4KAZ4HzgS3AfGPMR9ba5f6tLDAs2FhQE2gAvr5nCK3jIvxYEd4ws+5LmPWYd8fs+HYw+iXoNhocQf6tTURETkgnRagBBgBrrbXrAYwxbwMjgVM61Fhr+c07i3l/4VYAhnRK4pxOLfwbaKyF9bNg1t+8u2bHtoHh/4De12rBPBEROS4nS6hJATbXerwFGOinWgLGh4u21QSaa09L4y+XdfdfMdVh5pvHYdP3EJMCl/zdG2Y0zCQiIg3gZAk1dS0je9BlXcaYCcAEgLS0tMauya+stTw3ay2dk6N54+cDiY3wUy+IxwOrP4Vvn4KtCyC6tXdfpj7XK8yIiEiDOllCzRagTa3HqcC2A0+y1k4CJoH3ku6mKc0/lm4tZPWOEh4b3Z3EKD+EB48Hln8As5/wbjIZnw6XPgM9r1KYERGRRnGyhJr5QIYxph2wFRgHXO3fkvyj1OkiZ2shV06aB8BF3Vo1fRHrvoYZD8DOZZDYybfJ5GgIOln+uImISCA6Kb5lrLUuY8xtwAy8l3S/bK1d5ueymty0BVu4+93FNXs6PjKiK7GRTTjsVLwdPrkbVn4McW3h8v94d8zW1UwiItIETopQA2CtnQ5M93cd/uJye3j4I2+Osxaev7YPw5qylyZnGnx8F7iccN5DcNqvICS86d5fREROeSdNqDmVzVq1k69X7qTY6eKqAW3onhLXtIFm9QyYOh5S+8NlEyGxY9O9t4iIiI9CzQnu3ezN/HbqkprHj4zoRmiwo+kKyF8N034Oyd3h+g8hNLLp3ltERKQWhZoTUEWVm69W7uTHDQW8MW8jvdPiuH5QW+IiQps20Hg88M71EBQK4/6rQCMiIn6lUHOCKSit5LJn57CpoIywYAddW8fwwrV9aRHjh/kr676E/BXeCcFxbY58voiISCNSqDkBrMsvYdX2Yr5Zlc+UbO/Cyf+5oR9nZSYREtSEPTMAZQWw4n+w7D3Y8C1EtYQulzZtDSIiInVQqAlgHy7aysRZ61i5vbimrU1CBGP6tOG8Li2brpDyPbDyE1j2vnerA48LEtrDmXdC7+u0mJ6IiAQEhZoAVFheRYnTxR1vLzro2Fd3D2ma3pnKMu/2BkvehbUzwVPlXXtm0G3enbSTe1CzII6IiEgAUKgJEKVOF2HBDn7atJcrXph70PHRvVNIax7ZuIHG44H1X8GSd2DFx1BVCtGtYODN3iDTuo+CjIiIBCyFmibk8Vge/DCHMX1T6Z0WX9Ne6nTR+89fMKZvKoVlVTXto3qn8P7CrYQEGf5+Za/GK6x8Dyx8E+a/CHtyITwWul8O3a+AtqdrRWARETkhKNQ0ofW7Snnzh018tHgbSx++kJXbi/hp416ycwuodHn47w+bAOjaOobzOrfgNxd0YvyZ7YhrrK0OdiyDHyd5e2aqyqDNaXDug96Jv5onIyIiJxiFmkaUV1hOeaWb9klRLN68l5HPzgGguMLF4s17GT3xe9we72bhp7VPoLzSzdmZSdx+XgbBvmGmbimxDVuUx+1dAXjec5D7LQSHQ/exMOAX0Kpnw76XiIhIE1KoaUBLtxQSFR5Mu8RmzFu/mxte/hGny8NdQzPZXerc79xX5mzA7bFMv30wsZEhpMRFNG5xzmLvENMPz8OeDRCTCkMfgT7XQ2RC4763iIhIE1CoaUC3vfUTGS2ieGpsL176dgPNwoIZnBHH0zNXE+rreRndJ4X3ftrKB4u20bFFFFmtYxq3qB3LYf5LsGQKVJZAm4Ew9I/Q+VII0n9+ERE5eehbrYEUllexcXcZG3eX0fNPnwMw4az23HNBJ0b8+ztWbi/mvM4teGpsT+au201eYQU9Uht4aKmaqxJW/g9+fAk2fQ9BYdDtcuj/c0jt2zjvKSIi4mcKNQ1kZV7RQW3XDmxLaLCDuy/oxC9ezyYiNAhjDOP6p/Hh4q2M6ZPasEUUboEFr8KC16B0J8Snw/l/ht7XaohJREROego1DWT5AaHmoeFZpDX3bvB4XucWPHBxF4Z1SwbgjqEZ3DE0o2He2OOBDbNg/n9g1XSwFjKHeXtlOpwLjibeRkFERMRPFGoayPJt+0JNzzZx/OzMdjWPHQ7DL85q37BvWFEIi/7rnS+zey1ENocz7oC+N0F824Z9LxERkROAQk0DWbF9X6gJcTTiqrs7V8CPL8Lit70r/qYOgNEvQtZIrS0jIiKnNIWaBlDl9rB6ewmdWkazakcxwUENHGqshXVfwpxnYMNs78Tf6rVlWjfiSsMiIiInEIWaBrA+v5RKt4c+beNYtaOY5JjwhnlhjxtW/A+++zvkLYbo1jD0Yeh9PTRr3jDvISIicpJQqGkAq3YUA3Ddael0bBHNmL7HeVWTx+3duuDbp2D3GkjoACP+DT2uhODQBqhYRETk5KNQ0wDW7CgmyGHo0KLZ8S+mt+5r+PxB2LEUkrvD2FehywhtKikiInIEARdqjDFPAJcClcA64CZr7V5jTDqwAljlO3WetfYW33P6Aq8CEcB04A5rrW2qmldtLya9eSRhwccRPHau8IaZtV9AXBqMeRm6jgbTiJOORURETiKBuIjJF0A3a20PYDVwX61j66y1vXy3W2q1TwQmABm+27AmqxZYs7OEzJbRx/Zka2Huc/D8YNjyI1zwF7gt27sCsAKNiIjIUQu4UGOt/dxa6/I9nAccdoKKMaYVEGOtnevrnXkduKyRy6wxb/1ucneXknEsoaZ0N7w1DmbcBxnnw68Xwum/1qXZIiIixyDgQs0BfgZ8WutxO2PMQmPMN8aYwb62FGBLrXO2+NoOYoyZYIzJNsZk5+fnH3dxCzbuYdyked5FfFtG1e/JuXPg+TNh3Vdw0eMw7r+6oklEROQ4+GVOjTFmJpBcx6EHrLUf+s55AHABb/qO5QFp1trdvjk0HxhjugJ1jdHUOZ/GWjsJmATQr1+/455zs6vEWXO/09H21HjcMPtJ+OYxiG8H47/QWjMiIiINwC+hxlo79HDHjTE3AMOB86on/FprnYDTd3+BMWYdkIm3Z6b2EFUqsK0x6j7Q7pLKmvvpic2O7kmf/R5+nOS9PPuSpyDsGOfiiIiIyH4CbvjJGDMM+B0wwlpbVqs9yRgT5LvfHu+E4PXW2jyg2BhzmjHGANcDHzZFrfnF3p6aT+8YTEjQUXyU8573BppBt8HoSQo0IiIiDSjgLukG/g2EAV94M0rNpdtnAX8yxrgAN3CLtbbA95xfsu+S7k/Zfx5Oo9lV4iQ+MoQurY5ibZpVn3knBHceDuf/qfGLExEROcUEXKix1nY8RPs0YNohjmUD3RqzrrrkFztJij6KK5XylsDUn0FyD28PjRbSExERaXABN/x0IskvOYpQU1kK71wPEXFw1dsQepRzb0RERKReFGqOw64SJ82bHSHUfPUo7NkAo16AmFZNU5iIiMgpSKHmOJRVumkWdpihpJ0rYd5z0O9n0G7woc8TERGR46ZQcxyq3B5CD3fV0+wnICQSzvlD0xUlIiJyilKoOQ6VLg+hwYf4CHetgZxpMODnWilYRESkCSjUHIdKl+fQ69PMmwhBoTDo101blIiIyClKoeYYeTwWl8fW3VPjLIEl70C30RCV1PTFiYiInIIUao5RpdsDUHeoyZkKlcXQ96YmrkpEROTUpVBzjGpCTV3DTz9NhhZZ0GZAE1clIiJy6lKoOUaVrkP01BRuha3Z0H0MmLo2EBcREZHGoFBzjKoO1VOz8hPvzy4jmrgiERGRU5tCzTE6ZE/Nio8gsRMkZvihKhERkVPXUYcaY8yZxpibfPeTjDHtGq+swFcdava7pLusADZ+D12G+6kqERGRU9dRhRpjzB+B3wH3+ZpCgDcaq6gTQZ1XP636FKwbOivUiIiINLWj7akZBYwASgGstduA6MYq6kRQ5/DTyk8gJhVa9/ZTVSIiIqeuow01ldZaC1gAY0yzxivpxFAdasKqh5+shY1zoOO5uupJRETED4421LxjjHkBiDPG/AKYCbzYeGUFvurhp5Dqnpo9uVCxV700IiIifhJ8NCdZa580xpwPFAGdgIestV80amUB7qBLuvMWeX8q1IiIiPjFEUONMSYImGGtHQqc0kGmtoPm1OQtAUewdyVhERERaXJHHH6y1rqBMmNMbBPUc8JwHhhqdq6A5hkQHObHqkRERE5dRzX8BFQAS40xX+C7AgrAWnt7o1R1AqjpqakefspfAa37+LEiERGRU9vRThT+BHgQmA0sqHVrcMaYh40xW40xi3y3i2sdu88Ys9YYs8oYc2Gt9mG+trXGmN83Rl0HqnJbwNdTU1nqnSjcoktTvLWIiIjU4WgnCr9mjAkFMn1Nq6y1VY1XFk9ba5+s3WCMyQLGAV2B1sBMY0x1Pc8C5wNbgPnGmI+stcsbsT4qXW7A11OTv8zbqFAjIiLiN0cVaowxQ4DXgFzAAG2MMTdYa2c3XmkHGQm8ba11AhuMMWuBAb5ja6216321vu07t3FDTe1Luneu9DZqkrCIiIjfHO3w01PABdbas621ZwEXAk83XlncZoxZYox52RgT72tLATbXOmeLr+1Q7QcxxkwwxmQbY7Lz8/OPq8Ca4acgB+xcDsHhEJ9+XK8pIiIix+5oQ02ItXZV9QNr7Wq8+z8dE2PMTGNMTh23kcBEoAPQC8jDG6jA20N0IHuY9oMbrZ1kre1nre2XlJR0rOUD+65+Cgky3iufEjPBEXRcrykiIiLH7mivfso2xvwHmOx7fA3HMVHYt+bNERljXgQ+9j3cArSpdTgV2Oa7f6j2RlPp8hAa7MAYA7vWQNrAxn5LEREROYyj7an5JbAMuB24A+98lVsaoyBjTKtaD0cBOb77HwHjjDFhxph2QAbwIzAfyDDGtPNNZh7nO7dROV1u775P7ioo2qKhJxERET872p6aYOAZa+3foWaV4cZaZe5xY0wvvENIucDNANbaZcaYd/AGKhfwK9/CgBhjbgNmAEHAy9baZY1UW42ichcxESFQtBWsB+LaNvZbioiIyGEcbaj5EhgKlPgeRwCfA6c3dEHW2usOc+xR4NE62qcD0xu6lsMpqqgiOjwY9mz0NsSlNeXbi4iIyAGOdvgp3FpbHWjw3Y9snJJODEXlVcRGhMDeTd6GePXUiIiI+NPRhppSY0zNHgDGmH5AeeOUdGIoqvANP+3dCMYBMXVeRS4iIiJN5GiHn+4E3jXGbMM716U1cGWjVXUCKCqvIqZVjLenJiYVgo75CncRERFpAIftqTHG9DfGJFtr5wOdgSl4J+l+BmxogvoCVlF5FTERvjk1mk8jIiLid0cafnoBqPTdHwTcj3efpT3ApEasK6C5PZZip2vfnBrNpxEREfG7Iw0/BVlrC3z3rwQmWWunAdOMMYsat7TAVVzh3cszLtRCcZ56akRERALAkXpqgowx1cHnPOCrWseOdj7OSaeo3AVAst0FWK1RIyIiEgCOFEzeAr4xxuzCe7XTtwDGmI5AYSPXFrC+W7sLgAS7x9sQ3dKP1YiIiAgcIdRYax81xnwJtAI+t9ZWbxTpAH7d2MUFqn99tQaHgc5x3h4bIhL8W5CIiIgceQjJWjuvjrbVjVPOicHp8nD1wDRi7GJvQ0S8fwsSERGRo158T2rxWEuQMVDuG35SqBEREfE7hZpj4PFYjDFQvhdMEIRF+7skERGRU55CzTGwFhzVPTURcWCMv0sSERE55SnUHAOPtTgMvlCjoScREZFAoFBzDDwWHA6jUCMiIhJAFGqOgcda74iTQo2IiEjAUKg5BjVzair2KtSIiIgECIWaY6A5NSIiIoFHoeYYeKwlCA9UFEJ4nL/LERERERRqjonHQoS7xPtAPTUiIiIBQaGmnjwe7/ZXke5ib4NCjYiISEAIuFBjjJlijFnku+UaYxb52tONMeW1jj1f6zl9jTFLjTFrjTH/NKbxVsPz+Pb0jHAVeRsUakRERALCETe0bGrW2iur7xtjngIKax1eZ63tVcfTJgITgHnAdGAY8Glj1OfrqCHCo54aERGRQBJwPTXVfL0tVwBvHeG8VkCMtXautdYCrwOXNVZd1T014S5f1orQRGEREZFAELChBhgM7LDWrqnV1s4Ys9AY840xZrCvLQXYUuucLb62gxhjJhhjso0x2fn5+cdUlC/TEK7hJxERkYDil+EnY8xMILmOQw9Yaz/03b+K/Xtp8oA0a+1uY0xf4ANjTFegrvkztq73tdZOAiYB9OvXr85zjuSgOTW6pFtERCQg+CXUWGuHHu64MSYYGA30rfUcJ+D03V9gjFkHZOLtmUmt9fRUYFtD11xt3/BTEYTFQFDATUsSERE5JQXq8NNQYKW1tmZYyRiTZIwJ8t1vD2QA6621eUCxMeY03zyc64EP63rRhlA9UTisqkjzaURERAJIoHYzjOPgCcJnAX8yxrgAN3CLtbbAd+yXwKtABN6rnhrlyicA6+upCXMVaT6NiIhIAAnIUGOtvbGOtmnAtEOcnw10a+SygH09NeGuIohWT42IiEigCNThp4BVPacmrKpQPTUiIiIBRKGmnqpDTahCjYiISEBRqKknb6axvonCCjUiIiKBQqGmnjzW0owKHNalUCMiIhJAFGrqyWMhllLvA4UaERGRgKFQU08ejyXOlHgfaJ0aERGRgKFQU08ea4kxZd4H4bH+LUZERERqKNTUk8dCJBXeB6FR/i1GREREaijU1JPHWqIUakRERAKOQk09WWuJNNWhppl/ixEREZEaCjX15LHQrLqnJkw9NSIiIoFCoaaeqtepASBEPTUiIiKBQqGmnjweiDQVuB1hEBSQ+4GKiIickhRq6sk7Ubgct3ppREREAopCTT1ZC5HGiSs40t+liIiISC0KNfVUPafGE6JJwiIiIoFEoaaevKGmHLd6akRERAKKQk09eSw0M06FGhERkQCjUFNP1loiNfwkIiIScBRq6snbU1OBJ0Q9NSIiIoFEoaaePNYSTiWe4HB/lyIiIiK1+C3UGGPGGmOWGWM8xph+Bxy7zxiz1hizyhhzYa32Yb62tcaY39dqb2eM+cEYs8YYM8UYE9pYdXs8lmDc4AhprLcQERGRY+DPnpocYDQwu3ajMSYLGAd0BYYBzxljgowxQcCzwEVAFnCV71yA/wOettZmAHuA8Y1VtMdCEB5waDVhERGRQOK3UGOtXWGtXVXHoZHA29Zap7V2A7AWGOC7rbXWrrfWVgJvAyONMQY4F5jqe/5rwGWNVbfHWkJwgSOosd5CREREjkEgzqlJATbXerzF13ao9ubAXmut64D2gxhjJhhjso0x2fn5+cdUnMda9dSIiIgEoEb9ZjbGzASS6zj0gLX2w0M9rY42S90BzB7m/IMbrZ0ETALo169fneccifVYQoxboUZERCTANOo3s7V26DE8bQvQptbjVGCb735d7buAOGNMsK+3pvb5Dc7jcXvvaIduERGRgBKIw08fAeOMMWHGmHZABvAjMB/I8F3pFIp3MvFH1loLfA2M8T3/BuBQvUDHzVaHGvXUiIiIBBR/XtI9yhizBRgEfGKMmQFgrV0GvAMsBz4DfmWtdft6YW4DZgArgHd85wL8DviNMWYt3jk2/2m0wj1V3p8KNSIiIgHFb9/M1tr3gfcPcexR4NE62qcD0+toX4/36qjG5/HORzYKNSIiIgElEIefApp1+0KN5tSIiIgEFIWa+tLwk4iISEBSqKknTRQWEREJTAo19VUz/KS9n0RERAKJQk19+YafNFFYREQksCjU1Jdbi++JiIgEIoWaevLokm4REZGApFBTT0ahRkREJCAp1NSXR+vUiIiIBCKFmvpST42IiEhAUqipL1+oQZd0i4iIBBSFmvrS8JOIiEhAUqipL1+ocSjUiIiIBBSFmnoyvm0StKKwiIhIYFGoqa+a4acgPxciIiIitSnU1FP1OjUOh3pqREREAolCTX1ZTRQWEREJRAo19aQ5NSIiIoFJoaa+aubUKNSIiIgEEoWaejK2+pJuTRQWEREJJAo19eUbfnKop0ZERCSg+CXUGGPGGmOWGWM8xph+tdrPN8YsMMYs9f08t9axWcaYVcaYRb5bC197mDFmijFmrTHmB2NMeqPWronCIiIiAclf38w5wGjghQPadwGXWmu3GWO6ATOAlFrHr7HWZh/wnPHAHmttR2PMOOD/gCsbqW6MWxtaioiIBCK/9NRYa1dYa1fV0b7QWrvN93AZEG6MCTvCy40EXvPdnwqcZ4wxDVft/oz1Dj+hUCMiIhJQAnlOzeXAQmuts1bbK76hpwdrBZcUYDOAtdYFFALN63pBY8wEY0y2MSY7Pz//2KrSLt0iIiIBqdFCjTFmpjEmp47byKN4ble8w0g312q+xlrbHRjsu11XfXodL2Hrel1r7SRrbT9rbb+kpKT6/ULVb6aeGhERkYDUaN/M1tqhx/I8Y0wq8D5wvbV2Xa3X2+r7WWyM+S8wAHgd2AK0AbYYY4KBWKDgOMs/JIdvojBGl3SLiIgEkoAafjLGxAGfAPdZa+fUag82xiT67ocAw/FONgb4CLjBd38M8JW1ts6emgap0ePCbQ04AuqjExEROeX565LuUcaYLcAg4BNjzAzfoduAjsCDB1y6HQbMMMYsARYBW4EXfc/5D9DcGLMW+A3w+0at3bpxo14aERGRQOOXiSHW2vfxDjEd2P4X4C+HeFrfQ7xWBTC24ao7PGNduDX0JCIiEnA0hlJPDo8blz42ERGRgKNv53oy1oXLb2sWioiIyKHo27meHK16sNblpL+/CxEREZH9KNTUU//L7/J3CSIiIlIHDT+JiIjISUGhRkRERE4KCjUiIiJyUlCoERERkZOCQo2IiIicFBRqRERE5KSgUCMiIiInBYUaEREROSkYa62/a/ALY0w+sPEYnpoI7Grgck5k+jz2p89jH30W+ztZPo+21tokfxchUpdTNtQcK2NMtrW2n7/rCBT6PPanz2MffRb70+ch0vg0/CQiIiInBYUaEREROSko1NTfJH8XEGD0eexPn8c++iz2p89DpJFpTo2IiIicFNRTIyIiIicFhRoRERE5KSjU1IMxZpgxZpUxZq0x5vf+rqcpGGNeNsbsNMbk1GpLMMZ8YYxZ4/sZ72s3xph/+j6fJcaYPv6rvOEZY9oYY742xqwwxiwzxtzhaz9VP49wY8yPxpjFvs/jEV97O2PMD77PY4oxJtTXHuZ7vNZ3PN2f9TcGY0yQMWahMeZj3+NT9rMQ8QeFmqNkjAkCngUuArKAq4wxWf6tqkm8Cgw7oO33wJfW2gzgS99j8H42Gb7bBGBiE9XYVFzA3dbaLsBpwK98fwZO1c/DCZxrre0J9AKGGWNOA/4PeNr3eewBxvvOHw/ssdZ2BJ72nXeyuQNYUevxqfxZiDQ5hZqjNwBYa61db62tBN4GRvq5pkZnrZ0NFBzQPBJ4zXf/NeCyWu2vW695QNz/t3f3oXeOcRzH3x/z9MvKGJP2QENaediQyNMa/zBJTJuHLPlnpURJiAz5wz8oFHmYpzWt0UwrTTYPK4bZGvEXLUSWh5mnxHz8cV2H4zjm8PPb6Xefz6tO931f933uc53vH+d87+u67vuSdODOqenIs/2Z7bfr+reUP6+JDG48bPu7urlbfRmYBSyr5Z3xaMVpGXC6JO2k6o44SZOA2cBDdVsMaCwi+iVJTe8mAh+3bX9SywbRAbY/g/JHD0yo5QMTo9pdMANYxwDHo3a3bAS2AC8AHwBbbf9SD2n/zr/Ho+7/Bhi/c2s8ou4GrgV+rdvjGdxYRPRFkpredbuKyv3wfzYQMZI0FngauMr2th0d2qWsUfGwvd32dGASpTVzWrfD6rKx8ZB0NrDF9vr24i6HNj4WEf2UpKZ3nwCT27YnAZ/2qS799nmrG6Uut9TyxsdI0m6UhGax7Wdq8cDGo8X2VuAlylijcZJ2rbvav/Pv8aj79+avXZuj1UnAOZI2U7qmZ1FabgYxFhF9k6Smd28Ch9W7GXYH5gEr+lynflkBzK/r84Fn28ov10MjiQAAA5hJREFUrXf9nAB80+qWaYI65uFh4H3bd7btGtR47C9pXF0fAs6gjDNaA8yph3XGoxWnOcBqN+Tpn7avtz3J9sGU34bVti9mAGMR0U95ovC/IOksytXXGOAR27f3uUojTtISYCawH/A5cDOwHFgKTAE+Ai6w/VX907+XcrfUD8Bltt/qR71HgqSTgVeBd/hj3MQNlHE1gxiPoyiDXcdQLpCW2r5V0lRKa8W+wAbgEts/SdoTeIIyFukrYJ7tD/tT+5EjaSZwje2zBz0WETtbkpqIiIhohHQ/RURERCMkqYmIiIhGSFITERERjZCkJiIiIhohSU1EREQ0QpKaiB5I2i5pY9trh7O0S1og6dL/4XM3S9pvuOep59pL0gt1fW3bQ+EiIhohP2oRvfmxTgfQE9v3j2Rl/qMTgdcl7QN83zYnUUREI6SlJmIYakvKHZLeqK9Da/lCSdfU9SslvSdpk6Snatm+kpbXstfrg+yQNF7SKkkbJD1A2xxBki6pn7FR0gN1Mskxkh6V9K6kdyRd3aWOh9RJJ58ELgLWA0fX80zoPD4iYrRKUhPRm6GO7qe5bfu22T6e8vTgu7u89zpghu2jgAW17BZgQy27AXi8lt8MrLU9g/Io/SkAkqYBc4GTaovRduBiYDow0fYRto8EFnV+uO0P6nvWUyadfBy43PZ021s6j4+IGK3S/RTRmx11Py1pW97VZf8mYLGk5ZQpJgBOBs4HsL26ttDsDZwKnFfLV0r6uh5/OnAs8GaZfYEhysSZzwFTJd0DrARW7eA7TLD9paQjgQf/6QtHRIw2aamJGD7/zXrLbOA+SlKyvg7QVZfj3LFsJ+Cx2roy3fbhthfa/ho4mjJD9hXAQ395o3S/pHcpE7JupMxFtbJbV1VExGiWpCZi+Oa2LV9r3yFpF2Cy7TXAtcA4YCzwCqX7qDUB4he2t3WUnwnsU0/1IjCnNQamjsk5qN4ZtYvtp4GbgGM6K2d7AaW76zbgXGBlTYy6tSpFRIxa6X6K6M1QbeVoed5267buPSSto1wkXNjxvjHAk7VrScBdtrdKWggskrSJMoP3/Hr8LcASSW8DL1Nm/cb2e5JuBFbVROlnSsvMj/U8rQuU6/+m/qdRxtKcUs8bEdE4maU7YhgkbQaOs/1Fv+sSETHo0v0UERERjZCWmoiIiGiEtNREREREIySpiYiIiEZIUhMRERGNkKQmIiIiGiFJTURERDTCb7wxwBobtnFMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device:  cpu\n",
      "env_venv.observation_space.shape:  (24,) , len(obs_shape):  1\n",
      "env_venv.action_space:  Box(4,) , action_space.shape[0]:  4\n"
     ]
    }
   ],
   "source": [
    "#--------------- make_vec_envs ----------------\n",
    "## we continue with the same model, model Policy uses MLPBase, but with new environment env_venv\n",
    "\n",
    "device = torch.device(\"cpu\")\n",
    "print('device: ', device)\n",
    "\n",
    "seed = 0 \n",
    "num_processes=1  \n",
    "\n",
    "env_venv = make_vec_envs('BipedalWalker-v2', \\\n",
    "                    seed + 1000, num_processes,\n",
    "                    None, None, False, device='cpu', allow_early_resets=False)\n",
    "\n",
    "policy = policy.to(device)\n",
    "\n",
    "print('env_venv.observation_space.shape: ', env_venv.observation_space.shape, \\\n",
    "      ', len(obs_shape): ', len(env_venv.observation_space.shape))\n",
    "print('env_venv.action_space: ',  env_venv.action_space, \\\n",
    "      ', action_space.shape[0]: ', env_venv.action_space.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## No CUDA, only CPU\n",
    "\n",
    "def play_VecEnv(env, model, num_episodes):\n",
    "\n",
    "    obs = env.reset()\n",
    "    obs = torch.Tensor(obs)\n",
    "    obs = obs.float()\n",
    "        \n",
    "    recurrent_hidden_states = torch.zeros(1, model.recurrent_hidden_state_size)\n",
    "    \n",
    "    masks = torch.zeros(1, 1)\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "\n",
    "    render_func = get_render_func(env)\n",
    "        \n",
    "    for i_episode in range(1, num_episodes+1):     \n",
    "\n",
    "        time_start = time.time()\n",
    "        total_reward = np.zeros(num_processes)\n",
    "        timestep = 0\n",
    "\n",
    "        while True:\n",
    "        \n",
    "            with torch.no_grad():\n",
    "                value, action, _, recurrent_hidden_states = \\\n",
    "                    model.act(obs, recurrent_hidden_states, masks, \\\n",
    "                    deterministic=False) # obs = state\n",
    "\n",
    "            render_func()\n",
    "            \n",
    "            obs, reward, done, _ = env.step(action)\n",
    "            obs = torch.Tensor(obs)\n",
    "            obs = obs.float()\n",
    "\n",
    "            reward = reward.detach().numpy()\n",
    "            \n",
    "            masks.fill_(0.0 if done else 1.0)\n",
    "            \n",
    "            total_reward += reward[0]\n",
    "        \n",
    "            #if timestep < 800:\n",
    "            #    print('timestep: ', timestep, 'reward: ', reward[0])\n",
    "            \n",
    "            timestep += 1\n",
    "            \n",
    "            if timestep + 1 == 1600: ##   envs.max_steps:\n",
    "                break\n",
    "\n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(total_reward[0])\n",
    "        \n",
    "        avg_score = np.mean(scores_deque)\n",
    "                    \n",
    "        print('Episode {} \\tScore: {:.2f}, Avg.Score: {:.2f}, \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, total_reward[0], avg_score,  s//3600, s%3600//60, s%60))  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1 \tScore: 345.16, Avg.Score: 345.16, \tTime: 00:00:29\n",
      "Episode 2 \tScore: 312.54, Avg.Score: 328.85, \tTime: 00:00:27\n",
      "Episode 3 \tScore: 339.16, Avg.Score: 332.29, \tTime: 00:00:27\n",
      "Episode 4 \tScore: 303.27, Avg.Score: 325.03, \tTime: 00:00:27\n",
      "Episode 5 \tScore: 342.91, Avg.Score: 328.61, \tTime: 00:00:27\n"
     ]
    }
   ],
   "source": [
    "play_VecEnv(env=env_venv, model=policy, num_episodes=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#env_venv.close()\n",
    "#obs = env_venv.reset()\n",
    "#obs.close()\n",
    "#env_venv.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def load(model):\n",
    "#    model.base.actor.load_state_dict(torch.load('dir_save_VecEnv\\we0_actor_final.pth'))\n",
    "#    model.base.critic.load_state_dict(torch.load('dir_save_VecEnv\\we0_critic_final.pth'))\n",
    "#    model.base.critic_linear.load_state_dict(torch.load('dir_save_VecEnv\\we0_critic_linear_final.pth'))\n",
    "\n",
    "#def load_venv(model):\n",
    "#    model.base, ob_rms = \\\n",
    "#            torch.load('dir_save_VecEnv\\weights_venv_final.pth')\n",
    "\n",
    "\n",
    "# load(model=policy)    \n",
    "# load_venv(model=policy)\n",
    "# play_episode_parallelEnv(envs=envs, model=policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kernel-ml",
   "language": "python",
   "name": "kernel-ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
